
import os
from os.path import join 
import sys
from functools import partial
sys.path.append(os.path.join(os.getcwd(), '..')) #adds directory below as valid path
from  datetime import datetime, timedelta
dateformat = "%H-%M-%S"
from collections import deque
import traceback
from multiprocessing import Pool
from tqdm import tqdm

import scipy.constants as spc
from lmfit import Model, create_params

from scipy.integrate import odeint
from scipy.optimize import curve_fit

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

plt.rcParams['axes.grid'] = True
plt.rcParams['grid.linestyle'] = '--'

from MT_class_PID_new import MTdataHost
from global_folder.myplotsty import *
from global_folder.my_helpers import *

PUMP_FREQUENCY = 384228.6
REPUMP_FREQUENCY = 384228.6 + 6.56
SAMPLE_RATE = 2000
FREQVSVOLT = 221.0 
FREQVSCURR = 1.13

# TODO: find a better place for this
EXP_FOLDER =r'C:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\CATMeasurements'
MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'testPArun16')
WDATA_FOLDER =os.path.join(MEASURE_FOLDER, 'testPArun16.csv')

# TODO: maybe make a run analysis class out of this?
def dump():
	fig, ax = plt.subplots()
	plot_results(ax, df, max_freq= 384219., fmt='o', mfc='red', save_folder=MEASURE_FOLDER)
	plt.savefig(os.path.join(MEASURE_FOLDER, 'ratio_vs_freq.png'))

	
	collect_plots(MEASURE_FOLDER, os.path.join(MEASURE_FOLDER, 'collected_plots'), 'deloadPhase.png')
 
	#*-----------------------
	#* SINGLE RUN
 	#*----------------------- 
  
	MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'testPArun9')
	df = get_data_frame(MEASURE_FOLDER)
	df.dropna(inplace=True)
	#freqs = plot_results(df, 384201., save_folder=MEASURE_FOLDER)
 
	data = df.dropna()
	freqs = ((384201-PUMP_FREQUENCY)-(data['tempV']-df['tempV'].min())*FREQVSVOLT- (data['currV']-df['currV'].min())*FREQVSCURR)
	fig, ax = plt.subplots()
	plot_spline_fit(ax=ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'], 
				 s=0.1, save_folder=MEASURE_FOLDER, 
				 mfc='red', color='black', 
				 title='Trap Depth = 1.99 K')
 
	# *-----------------------
	# * MULTIPLE RUN COMPARISON
 	# *-----------------------
	folders = [os.path.join(EXP_FOLDER, path ) for path in ['PArunHalfVarDet1', 'testPAVaryingCATampl']]
	dfs = [get_data_frame(measure_folder) for measure_folder in folders]

	dfs = [get_data_frame(measure_folder) for measure_folder in folders]

	max_freqs = [384182.6]*len(dfs)
	zipped_data = list(zip(dfs, max_freqs))
	for i, (df, max_freq)  in enumerate(zipped_data[:]):
		fig, ax = plt.subplots()
		data = df.dropna()
		freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'], scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.00, ms=5, save_folder=join(folders[i]))

	#*-----------------------
	#* PARSING WAVEMETER DATA
 	#*-----------------------

	MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'testPArun14')
	WDATA_FOLDER =os.path.join(MEASURE_FOLDER, 'testPArun14.csv')
	freq_data, max_freq, min_freq = add_wavemeter_data('', WDATA_FOLDER)
	data = freq_data[:]
	levels = staircase_fit(data)

	data = get_data_frame(MEASURE_FOLDER)
	data.dropna(inplace=True)
	freqs = ((max_freq)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
	plt.plot(freqs)
 
	#*-----------------------
	#* GETTING DEPTH_RATIO DATAFRAME
 	#*-----------------------
	MEASURE_FOLDER = r'C:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\CATMeasurements\relScatRate'
	depth_ratios_df = get_rel_scattering_df(MEASURE_FOLDER)
	df_slice = depth_ratios_df[(depth_ratios_df['pa1']==1.85) & (depth_ratios_df['pd1']==84) & (depth_ratios_df['pd2'] == 84)]
	x = df_slice['pa2']
	y = df_slice['depth_ratio']
	plt.plot(x,y, 'o')

	#*-----------------------
	#* MEGA_RUN
 	#*-----------------------	
	MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'MegaRuns', 'testPArunMega3')
	df = get_data_frame(MEASURE_FOLDER,
						plot=False,
						cache_all=True)
	dfc= df.copy()
	#df.dropna(inplace=True)

	groupbyKey = 'pump_reference'
	titleKey = 'pump_AOM_freq'

	df_grouped = df.groupby(by=groupbyKey)
	min_ratios = df_grouped['ratio'].min()

	groups = dict(list(df_grouped))
	dfs = [df for df in groups.values()]

	# plotting ratio vs freq
	max_freqs = [384182.5]*len(dfs)
	zipped_data = list(zip(dfs, max_freqs))
	fig, ax = plt.subplots()
	for i, (df, max_freq)  in enumerate(zipped_data[:]):
		data = df
		freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.0, ms=5, figsize=(10, 10), label=f"{groupbyKey} = { data.iloc[10][groupbyKey] :.2f}", linewidth=2.5)

	plt.legend()
	plt.savefig(os.path.join(MEASURE_FOLDER, 'lossFeatures.png'))
	plt.title(f'Loss Features, {titleKey} = {data[titleKey].mean():.2f}', **titledict)
	plt.show()
	plt.close()
	#---------------------------------------------------
	x = [df[groupbyKey].mean() for df in dfs]
	y = [(df['ratio'].max() - df['ratio'].min()) for df in dfs]
	plt.plot( x, y ,'-o')
	plt.xlabel(groupbyKey)
	plt.ylabel(r'SNR $ = V_{ss, off} - V_{ss, on}$ ')
	plt.title(f'SNR Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict)
	plt.show()
	plt.savefig(join(MEASURE_FOLDER, 'SNRplot.png'), dpi=200)
	plt.close()

	#---------------------------------------------------
	# Plotting 2 body decay rate
	#dfs = [df for df in groups.values()]

	for i, df  in enumerate(dfs[:]):
			data = df
			freqs = ((384182.5-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
			betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
			freqs = sorted(freqs)
			plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{groupbyKey}={data.iloc[10][groupbyKey]:.2f}")
	plt.legend()
	plt.xlabel(r'$\Delta$ (GHz)')
	plt.ylabel(r'$\beta_{\mathrm{eff}}$ ')
	plt.savefig(join(MEASURE_FOLDER, 'betaVsFreq.png'), dpi=200) 
	plt.title(f'2-body Decay Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict) 
	plt.show()
	plt.close()
	#*-----------------------
	#* MULTIPLE MEGARUN
	#*-----------------------
 
	folders = [os.path.join(EXP_FOLDER, 'MegaRuns', path ) for path in ['testPArunMega7', 'testPArunMega8']]

	dfs_mega = [get_data_frame(measure_folder, cache_all=True) for measure_folder in folders]

	groupbyKey = 'pump_reference'
	titleKey = 'pump_AOM_freq'

	dfs_grouped = [df_mega.groupby(by=groupbyKey) for df_mega in dfs_mega]
	min_ratios = [df_grouped['ratio'].min() for df_grouped in dfs_grouped]

	groupss = [dict(list(df_grouped)) for df_grouped in dfs_grouped]
	dfs = [ [df for df in groups.values()] for groups in groupss]
 
	for row in dfs:
		data = row[3]
	
		freqs = ((384182.5-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
		freqs = sorted(freqs)
		plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{titleKey}={data.iloc[10][titleKey]:.2f}")
		plt.title(f'{groupbyKey} = {data.iloc[10][groupbyKey]:.2f}')
	plt.legend()
	
	#*-----------------------
	#* FULL RUNS
	#*-----------------------
	
	MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'testPArunFull4')
	df = get_data_frame(MEASURE_FOLDER)
	df.dropna(inplace=True)

	dfc = df.copy()
	df_grouped = df.groupby(by=['pump_reference', 'pump_AOM_freq'])
	groups = dict(list(df_grouped))
	dfs = [df for df in groups.values()]
 
	max_freqs = [384182.8]*30
	zipped_data = list(zip(dfs, max_freqs))
	fig1, ax1s = plt.subplots(4)
	fig2, ax2s = plt.subplots(5)
	for i, (df, max_freq)  in enumerate(zipped_data[:]):
		j1 = i%4
		j2 = i//4
		
		data = df.dropna()
		freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-0.0)*FREQVSCURR)
		ax2s[j2] = plot_spline_fit(ax2s[j2], x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i%4}', mfc=f'C{i%4}',color=f'C{i%4}', s=0.0, ms=5, figsize=(5, 25), linewidth=1.5, label=f"Detuning  = { 180-2*df.iloc[10]['pump_AOM_freq'] :.2f}", fig=fig2)
		ax2s[j2].set_title(f"Pump Amplituide = { df.iloc[10]['pump_reference'] :.2f}", **titledict)
		ax2s[j2].legend()
		
		ax1s[j1] = plot_spline_fit(ax1s[j1], x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i//4}', mfc=f'C{i//4}',color=f'C{i//4}', s=0.0, ms=5, figsize=(5, 25), linewidth=1.5, label=f"Pump Amplitude = { df.iloc[10]['pump_reference'] :.2f}", fig=fig1)
		ax1s[j1].set_title(f"Detuning  = { 180-2*df.iloc[10]['pump_AOM_freq'] :.2f}", **titledict)
		ax1s[j1].legend()
		
	fig1.tight_layout()
	fig2.tight_layout()

	fig1.savefig(os.path.join(MEASURE_FOLDER, 'lossFeaturesDet.png'))
	plt.show()
	plt.close()

	fig2.savefig(os.path.join(MEASURE_FOLDER, 'lossFeaturesPA.png'))
	plt.show()
	plt.close()
	
	SNRdata = df_grouped['ratio'].max() - df_grouped['ratio'].min()
	SNRdf = SNRdata.reset_index()
	SNRdf.columns = ['pump_reference', 'pump_AOM_freq', 'SNR']
	pivot_table = SNRdf.pivot('pump_reference', 'pump_AOM_freq', 'SNR')
	xticklabels = [f'{180-2*x:.2f}' for x in pivot_table.columns]
	yticklabels = [f'{y:.2f}' for y in pivot_table.index]
	sns.heatmap(pivot_table, annot=True, fmt='.2f', xticklabels=xticklabels, yticklabels=yticklabels)
	plt.xlabel("Detuning (MHz)")
	plt.ylabel("Pump Reference")
	plt.grid()
	plt.savefig(os.path.join(MEASURE_FOLDER, 'heatmap.png'))
	plt.show()
	plt.close()

	# fig, ax = plt.subplots()
	# for i, (df, max_freq)  in enumerate(zipped_data[:]):
	# 	data = df.dropna()
	# 	freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		
	# 	ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.0, ms=5, figsize=(10, 10), linewidth=2.5)
		
	# 	plt.title(f"Pump Amplituide = { df.iloc[10]['pump_reference'] :.2f}, \
	# 				sDetuning  = { 180-2*df.iloc[10]['pump_AOM_freq'] :.2f}", **titledict)

	# 	plt.legend()

	# 	plt.savefig(os.path.join(MEASURE_FOLDER, f'lossFeatures{i}.png'))
	# 	plt.show()
		
	# 	plt.close()
	# 	fig, ax = plt.subplots()



def freq_misc():
	WDATA_FOLDER = r'C:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\CATMeasurements\CATcurrTestrun3.csv'
	freq_data = add_wavemeter_data('', WDATA_FOLDER)
	levels = staircase_fit(freq_data[0], peak_height=0.2, distance=50, data_offset=1, window_size=1) 

	plt.close()
 
	x = np.linspace(0, 4.9, 25)
	y = levels
 
	plt.plot(levels, '-o')
	plt.title('Levels plot')
	
	m, b, fit_line = my_linear_fit(x, y)
def save_fit_results(run_path, plot=False, bkfile=False, 
					 CATbaseline=True, MOTbaseline=True,
					 initRFit=True, loadFit=True,deloadFit=True,reloadFit=True,
					 storeFitResults=True):
	filename = os.path.join(run_path, 'data.csv')
	bkfilename = os.path.join(run_path, 'data_oldPD.csv')
	settingsname = os.path.join(run_path, 'Settings.txt')
	
	dataHost = MTdataHost(SAMPLE_RATE)
	dataHost.loadCATdata(fileName=filename, settingsName=settingsname)
	
	if bkfile:
		dataHost.CATbackgroundData(bkfilename)
	
	#dataHost.setAllCAT(0.002)
	if CATbaseline:
		dataHost.setCATbaseline(0.002)
	if MOTbaseline:
		dataHost.setBaseline(0.002)
	if loadFit:
		dataHost.setLoading(0.002)
	if initRFit and loadFit:
		dataHost.initFit, dataHost.initX = dataHost.setInitialLoad(0.002)
	if deloadFit:
		dataHost.setDeloading(0.002)
		dataHost.plotDeloadFit(run_path)# TODO: currently just stores the deloading times and voltages
	if reloadFit:
		dataHost.setReloadVolt(0.002)

	if CATbaseline and MOTbaseline and loadFit and reloadFit:
	# steady state ratio fraction
		dataHost.ratio = dataHost.reloadVolt / dataHost.motSS

		dataHost.ratioErr = dataHost.ratio * ((dataHost.reloadVoltErr/dataHost.reloadVolt)**2 + (dataHost.motSSErr/dataHost.motSS)**2)**(0.5)
	
		# if abs(dataHost.ratioErr / dataHost.ratio) > 0.1:
		#     dataHost.ratioErr = abs(0.015*dataHost.ratio)
		if dataHost.ratioErr < 0.001:
			dataHost.ratioErr = 0.001
		if dataHost.ratio < 0:
			dataHost.ratio = 0
			
		# TODO: this information is useless
	print('File loaded: RFmin = {} MHz, t_mt = {:.3f} s.'.format(dataHost.settings['fmin'], dataHost.settings['wait_mtrap']))
	
	resultDict = dataHost.getResults(run_path, store=storeFitResults)
 
	if plot:
		dataHost.storeFits(run_path, combined=True, separate=True)
  
	return resultDict, dataHost.settings
		
def get_timestamp(run_path):
	timestamp = datetime.strptime(os.path.split(run_path)[-1].split('_')[0], dateformat)
	return timestamp

def extract_fit(run_path, plot=True, cache_failed=True, cache_all=True):
	"""Gather relevant data from each measurement run

	Args:
		run_path : absolute path to the run directory
		plot (bool, optional): plot fits. Defaults to True.
		cache_failed (bool, optional): Cache failed fits. If false, refit. Doesn't refit non-failed fits. Defaults to True.
		cache_all (bool, optional): If false, ignore any cached fit_results. Defaults to True.

	Returns:
		a 3-tuple (fit_results, settings, timestamp)
	"""
	fit_results, settings, timestamp = {}, {}, None
	
	if not os.path.isdir(run_path):
		return fit_results, settings, timestamp  # directory is not a run directory

	try:
		timestamp = get_timestamp(run_path)
  
  	# TODO: specify which error to catch
	except Exception as e:
		print("Error extracting timestamp from: ", run_path)
		print(traceback.format_exc())
	
	MAT_fit_cache_path = os.path.join(run_path, 'resultDict.txt')
 
	if not os.path.exists(MAT_fit_cache_path) or not cache_all:
		
		try: 
			fit_results, settings = save_fit_results(run_path, plot=plot)
		except Exception as e:
			print(traceback.format_exc())
			print("Fitting ERROR at ", os.path.basename(run_path), '\n')

			with open(MAT_fit_cache_path, 'w') as f:
				f.write(str('MAT fit failed'))
		
	else:
		print("Accessing cached results from :", os.path.basename(run_path))
  
		fit_results = open(MAT_fit_cache_path, 'r').read()

		if fit_results == 'MAT fit failed':
			if not cache_failed:
				# fit regardless of cached result
				try: 
					fit_results, settings = save_fit_results(run_path, plot=plot)
				except Exception as e:
					print(traceback.format_exc())
					print("Fitting ERROR at ", os.path.basename(run_path), '\n')
	
					with open(MAT_fit_cache_path, 'w') as f:
						f.write(str('MAT fit failed'))	
			else:
				print("Failed fit at :", os.path.basename(run_path))
				fit_results = {}

		else:
			fit_results = eval(open(MAT_fit_cache_path, 'r').read())
   
			settingsname = os.path.join(run_path, 'Settings.txt')
			settings = eval(open(settingsname, 'r').read())
   
	return fit_results, settings, timestamp


def get_row(run_path, **kwargs):
			
   fit_results, settings, timestamp = extract_fit(run_path, **kwargs)
   row = {**fit_results, **settings, **{'timestamp':timestamp}}
   
   return row

   
def get_data_frame(data_dir, parallel=True, in_process_run=False, **kwargs):
   
	run_path_arr = []
	rows = []
	
	for relative_path in os.listdir(data_dir):
		run_path_arr.append(os.path.join(data_dir, relative_path))
  
	if in_process_run:
		run_path_arr.pop()

	run_path_arr = sorted(run_path_arr)
	
	if parallel:
		with Pool(4) as p:
			rows = list(tqdm(p.imap(partial(get_row, **kwargs), run_path_arr), total=len(run_path_arr)))
	
	else:
		for run_path in tqdm(run_path_arr):
		   rows.append(get_row(run_path, **kwargs))
	return pd.DataFrame.from_dict(rows)

def add_wavemeter_data(df, wmeter_csv_path, window_size=100, num_rows=50):


	"""Extract unique frequnecy values from wavemeter data
	Returns:
		unique_levels (list): unique frequency values in wavemeter data
	"""
 
	# TODO: modify dataframe in place with frequency data
 
	wdata = pd.read_csv(wmeter_csv_path, skiprows=2)
	wdata.dropna(inplace=True)
	freq_data = np.array(wdata.iloc[:, 0])
	try:
		freq_data = np.array([float(item) for item in freq_data if item.replace('.','').isdigit()])
	except Exception as e:
		print(e)
 
	max_freq = freq_data.max()
	min_freq = freq_data.min()

	return freq_data, max_freq, min_freq

def plot_results(ax, dfs, max_freq, min_freq=0.0, mfc='red', fmt='o', ms=5, save_folder=False, xscale=1.0, yscale=1.0, **kwargs):
	FREQVSVOLT = 221.0 
	FREQVSCURR = 1.13
	
	if not type(dfs) == list:
		freqs = ((max_freq-PUMP_FREQUENCY)-(dfs.dropna()['tempV']-dfs.dropna()['tempV'].min())*FREQVSVOLT- (dfs.dropna()['currV']-dfs.dropna()['currV'].min())*FREQVSCURR)*xscale
		dfs=[dfs]
  
	plt.gcf().set_dpi(300)
	
	for df in dfs:
		df = df.dropna()
		ax.errorbar(freqs, 
			   df['ratio']*yscale,
			   yerr=df['ratioErr'],
			   fmt=fmt, mfc=mfc, color='black', ms=ms, **kwargs)
		ax.set_ylabel(r'$\mathbf{\frac{V_{ss, cat}}{V_{ss}}} $ ', **labeldict)
		ax.set_xlabel(r'$\Delta $ (GHz)', **labeldict)
	
	if save_folder:
		plt.savefig(os.path.join(save_folder, 'ratio_vs_freq.png'))
	
	return freqs, ax
  
	#return plt.gca(), plt.gcf()
	#plt.show()

def plot_spline_fit(ax, x, y, s=1, yerr=None, color='black', scolor='black',figsize=(12,5), save_folder=None, title='',alpha=0.5,dpi=200, label='plot', fig=None,**kwargs):
	from scipy.interpolate import splev, splrep
	xnew = np.linspace(min(x), max(x), 3*len(x) )

	y = [b for a,b in sorted(zip(x,y), key=lambda pair: pair[0])]
	if yerr is not None:
		yerr = [b for a,b in sorted(zip(x,yerr), key=lambda pair: pair[0])]
 
	x = sorted(x)


	spl = splrep(x, y, s=s)
	ynew = splev(xnew, spl)

	if fig is None:
		plt.gcf().set_dpi(dpi)
		plt.gcf().set_size_inches(figsize)
	else:
		fig.set_dpi(dpi)
		fig.set_size_inches(figsize)
	if yerr is not None:
		ax.errorbar(x, y, yerr=yerr, fmt='o', **kwargs)
	else:
		ax.plot(x,y, 'o', **kwargs)
	ax.plot(xnew, ynew, '-', color=scolor, alpha=alpha, label=label, **kwargs)
 
	ax.set_ylabel(r'$\mathbf{\frac{V_{ss, cat}}{V_{ss}}} $ ', **labeldict)
	ax.set_xlabel(r'$\Delta $ (GHz)', **labeldict)
 
	ax.set_title(title, **titledict)
	if save_folder:
		plt.savefig(os.path.join(save_folder, 'spline_ratio_vs_freq.png'))
	
	return ax
 
def plot_polyfit(x_data, y_data, spline_degree):
	
	coefficients = np.polyfit(x_data, y_data, spline_degree)

	x_interp = np.linspace(min(x_data), max(x_data), 100)
	y_interp = np.polyval(coefficients, x_interp)

	plt.scatter(x_data, y_data, label='Original Data')
	plt.plot(x_interp, y_interp, label='Polynomial Interpolation (Degree={})'.format(spline_degree))

def collect_plots(source, destination, plot_name):
	print(f'Collecting plots from {os.path.basename(source)}')
	import shutil

	os.makedirs(destination, exist_ok=True)

	plot_files = []
	for root, dirs, files in os.walk(source):
		
		for file in files:
			if file == plot_name:
				plot_files.append(os.path.join(root, file))

	for i, plot_file in enumerate(plot_files, start=0):
		new_filename = f'{i}{plot_name}'
		destination_path = os.path.join(destination, new_filename)
		shutil.copy(plot_file, destination_path)

def create_GIF(images_folder, image_name):
	import imageio
	with imageio.get_writer(os.path.join(images_folder, f'{image_name}movie.gif'), mode='I', duration=0.5) as writer:
		for filename in os.listdir(images_folder):
			if image_name in filename:
				image = imageio.imread(os.path.join(images_folder, filename))
				writer.append_data(image)

def staircase_fit(data, peak_height=0.1, distance=100, data_offset=1, window_size=1, inc_final_peak=True):
	def moving_average(arr, window_size):
		weights = np.ones(window_size) / window_size
		return np.convolve(arr, weights, mode='valid')

	convdata1 = moving_average(data, window_size)
	convdata2 = moving_average(data[data_offset+1:], window_size )
	final = convdata1[:len(convdata2)]-convdata2
	# plt.plot(data)
	# plt.plot(convdata1)
	# plt.plot(convdata2)
	# plt.show()
	# plt.plot(final)

	
	from scipy.signal import find_peaks
	x= final
	peaks, _ = find_peaks(x, height=peak_height, distance=distance)
	peaks = np.insert(peaks, 0, 0)
	levels = []
	plot_arr = []
	for i, peak in enumerate(peaks):
		
		if i < len(peaks) - 1:
			temp = data[ peaks[i]:peaks[i+1] ]
			plot_arr.extend( np.ones_like(temp)*np.mean(temp))
			levels.append(np.mean(temp))
	if inc_final_peak:
		temp = data[peaks[-1]:]
		levels.append(np.mean(temp))
		plot_arr.extend( np.ones_like(temp)*np.mean(temp))
  
	plt.plot(x)
	plt.plot(peaks, x[peaks], "x")
	plt.plot(np.zeros_like(x), "--", color="gray")
	plt.show()
	
	plt.plot(data)
	plt.plot(np.ravel((plot_arr)))
	plt.show()
	plt.close()
	plt.plot(np.array(levels)[np.where(abs(np.diff(levels))>0.05)[0]], 'o', ms=5)
 
	return levels

def load_single_run(run_path):
	filename = os.path.join(run_path, 'data.csv')
	bkfilename = os.path.join(run_path, 'data_oldPD.csv')
	settingsname = os.path.join(run_path, 'Settings.txt')
	
	dh1 = MTdataHost(SAMPLE_RATE)
	dh1.loadCATdata(fileName=filename, settingsName=settingsname)
	
	return dh1
	
def load_mega_run(MEASURE_FOLDER, groupbyKey, titleKey, plot=True, save_plots=False, max_freq=384182.5, **kwargs):
    # TODO: use the plot flag to do something?
    
	df = get_data_frame(MEASURE_FOLDER,
						**kwargs)
	dfc= df.copy()
	#df.dropna(inplace=True)

	df_grouped = df.groupby(by=groupbyKey)
	min_ratios = df_grouped['ratio'].min()

	groups = dict(list(df_grouped))
	dfs = [df for df in groups.values()]

	# plotting ratio vs freq
	max_freqs = [max_freq]*len(dfs)
	zipped_data = list(zip(dfs, max_freqs))
	fig1, ax = plt.subplots()
	for i, (df, max_freq)  in enumerate(zipped_data[:]):
		data = df
		freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.0, ms=5, figsize=(10, 10), label=f"{groupbyKey} = { data.iloc[10][groupbyKey] :.2f}", linewidth=2.5)


	plt.legend()
	
	plt.title(f'Loss Features, {titleKey} = {data[titleKey].mean():.2f}', **titledict)
	plt.show()

	#---------------------------------------------------
	fig2 = plt.figure(2)
	x = [df[groupbyKey].mean() for df in dfs]
	y = [(df['ratio'].max() - df['ratio'].min()) for df in dfs]
	plt.plot( x, y ,'-o')
	plt.xlabel(groupbyKey)
	plt.ylabel(r'SNR $ = V_{ss, off} - V_{ss, on}$ ')
	plt.title(f'SNR Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict)
	plt.show()
	
	#---------------------------------------------------------
	fig3=plt.figure(3)
	for i, df  in enumerate(dfs[:]):
		data = df
		freqs = ((384182.5-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
		freqs = sorted(freqs)
		plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{groupbyKey}={data.iloc[10][groupbyKey]:.2f}")
	plt.legend()
	plt.xlabel(r'$\Delta$ (GHz)')
	plt.ylabel(r'$\beta_{\mathrm{eff}}$ ')
	plt.title(f'2-body Decay Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict) 
	plt.show()

	if save_plots:
		fig1.savefig(os.path.join(MEASURE_FOLDER, 'lossFeatures.png'))
		fig2.savefig(join(MEASURE_FOLDER, 'SNRplot.png'), dpi=200)
		fig3.savefig(join(MEASURE_FOLDER, 'betaVsFreq.png'), dpi=200) 
	return dfc


if __name__ == '__main__':
	# run_path = r"C:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\CATMeasurements\testPArun9\16-53-10"
	# filename = os.path.join(run_path, 'data.csv')
	# bkfilename = os.path.join(run_path, 'data_oldPD.csv')
	# settingsname = os.path.join(run_path, 'Settings.txt')
	
	# dh1 = MTdataHost(SAMPLE_RATE)
	# dh1.loadCATdata(fileName=filename, settingsName=settingsname)
	
	# dh1.setAllCAT(0.002)
 
 	#dh1.CATbackgroundData(bkfilename)
	
	pass


folders = [join(EXP_FOLDER, 'RepumpSizeTrapDepthStudy', path ) for path in ['PAsmallRepumpSizeTest', 'PAsmallRepumpSizeTest2', 'PAsmallRepumpSizeTest3', 'PAwideRepumpSizeTest', 'PAwideRepumpSizeTest2']]

dfs = [get_data_frame(measure_folder, cache_all=False, plot=False) for measure_folder in folders][1:]
labels = ['Narrow Beam' , 'Narrow Beam', 'Narrow Beam', 'Wide Beam', 'Wide Beam' ][1:]
max_freqs = [384182.6]*len(dfs)
zipped_data = list(zip(dfs, max_freqs))
for i, (df, max_freq)  in enumerate(zipped_data[:]):
    data=df
    freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
    betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
    freqs = sorted(freqs)
    plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{labels[i]}")
    plt.xlabel(r'$\Delta$ (GHz)')
    plt.ylabel(r'$\beta$')
    
plt.legend()

100%|██████████| 24/24 [00:11<00:00,  2.10it/s]
100%|██████████| 23/23 [00:11<00:00,  2.00it/s]
100%|██████████| 23/23 [00:12<00:00,  1.80it/s]
 12%|█▎        | 3/24 [00:13<01:20,  3.83s/it]


import os
from os.path import join 
import sys
from functools import partial
sys.path.append(os.path.join(os.getcwd(), '..')) #adds directory below as valid path
from  datetime import datetime, timedelta
dateformat = "%H-%M-%S"
from collections import deque
import traceback
from multiprocessing import Pool
from tqdm import tqdm

import scipy.constants as spc
from lmfit import Model, create_params

from scipy.integrate import odeint
from scipy.optimize import curve_fit

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

plt.rcParams['axes.grid'] = True
plt.rcParams['grid.linestyle'] = '--'

from MT_class_PID_new import MTdataHost
from global_folder.myplotsty import *
from global_folder.my_helpers import *

PUMP_FREQUENCY = 384228.6
REPUMP_FREQUENCY = 384228.6 + 6.56
SAMPLE_RATE = 2000
FREQVSVOLT = 221.0 
FREQVSCURR = 1.13

# TODO: find a better place for this
EXP_FOLDER =r'C:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\CATMeasurements'
MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'testPArun16')
WDATA_FOLDER =os.path.join(MEASURE_FOLDER, 'testPArun16.csv')

# TODO: maybe make a run analysis class out of this?
def dump():
	fig, ax = plt.subplots()
	plot_results(ax, df, max_freq= 384219., fmt='o', mfc='red', save_folder=MEASURE_FOLDER)
	plt.savefig(os.path.join(MEASURE_FOLDER, 'ratio_vs_freq.png'))

	
	collect_plots(MEASURE_FOLDER, os.path.join(MEASURE_FOLDER, 'collected_plots'), 'deloadPhase.png')
 
	#*-----------------------
	#* SINGLE RUN
 	#*----------------------- 
  
	MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'testPArun9')
	df = get_data_frame(MEASURE_FOLDER)
	df.dropna(inplace=True)
	#freqs = plot_results(df, 384201., save_folder=MEASURE_FOLDER)
 
	data = df.dropna()
	freqs = ((384201-PUMP_FREQUENCY)-(data['tempV']-df['tempV'].min())*FREQVSVOLT- (data['currV']-df['currV'].min())*FREQVSCURR)
	fig, ax = plt.subplots()
	plot_spline_fit(ax=ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'], 
				 s=0.1, save_folder=MEASURE_FOLDER, 
				 mfc='red', color='black', 
				 title='Trap Depth = 1.99 K')
 
	# *-----------------------
	# * MULTIPLE RUN COMPARISON
 	# *-----------------------
	folders = [os.path.join(EXP_FOLDER, path ) for path in ['PArunHalfVarDet1', 'testPAVaryingCATampl']]
	dfs = [get_data_frame(measure_folder) for measure_folder in folders]

	dfs = [get_data_frame(measure_folder) for measure_folder in folders]

	max_freqs = [384182.6]*len(dfs)
	zipped_data = list(zip(dfs, max_freqs))
	for i, (df, max_freq)  in enumerate(zipped_data[:]):
		fig, ax = plt.subplots()
		data = df.dropna()
		freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'], scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.00, ms=5, save_folder=join(folders[i]))

	#*-----------------------
	#* PARSING WAVEMETER DATA
 	#*-----------------------

	MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'testPArun14')
	WDATA_FOLDER =os.path.join(MEASURE_FOLDER, 'testPArun14.csv')
	freq_data, max_freq, min_freq = add_wavemeter_data('', WDATA_FOLDER)
	data = freq_data[:]
	levels = staircase_fit(data)

	data = get_data_frame(MEASURE_FOLDER)
	data.dropna(inplace=True)
	freqs = ((max_freq)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
	plt.plot(freqs)
 
	#*-----------------------
	#* GETTING DEPTH_RATIO DATAFRAME
 	#*-----------------------
	MEASURE_FOLDER = r'C:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\CATMeasurements\relScatRate'
	depth_ratios_df = get_rel_scattering_df(MEASURE_FOLDER)
	df_slice = depth_ratios_df[(depth_ratios_df['pa1']==1.85) & (depth_ratios_df['pd1']==84) & (depth_ratios_df['pd2'] == 84)]
	x = df_slice['pa2']
	y = df_slice['depth_ratio']
	plt.plot(x,y, 'o')

	#*-----------------------
	#* MEGA_RUN
 	#*-----------------------	
	MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'MegaRuns', 'testPArunMega3')
	df = get_data_frame(MEASURE_FOLDER,
						plot=False,
						cache_all=True)
	dfc= df.copy()
	#df.dropna(inplace=True)

	groupbyKey = 'pump_reference'
	titleKey = 'pump_AOM_freq'

	df_grouped = df.groupby(by=groupbyKey)
	min_ratios = df_grouped['ratio'].min()

	groups = dict(list(df_grouped))
	dfs = [df for df in groups.values()]

	# plotting ratio vs freq
	max_freqs = [384182.5]*len(dfs)
	zipped_data = list(zip(dfs, max_freqs))
	fig, ax = plt.subplots()
	for i, (df, max_freq)  in enumerate(zipped_data[:]):
		data = df
		freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.0, ms=5, figsize=(10, 10), label=f"{groupbyKey} = { data.iloc[10][groupbyKey] :.2f}", linewidth=2.5)

	plt.legend()
	plt.savefig(os.path.join(MEASURE_FOLDER, 'lossFeatures.png'))
	plt.title(f'Loss Features, {titleKey} = {data[titleKey].mean():.2f}', **titledict)
	plt.show()
	plt.close()
	#---------------------------------------------------
	x = [df[groupbyKey].mean() for df in dfs]
	y = [(df['ratio'].max() - df['ratio'].min()) for df in dfs]
	plt.plot( x, y ,'-o')
	plt.xlabel(groupbyKey)
	plt.ylabel(r'SNR $ = V_{ss, off} - V_{ss, on}$ ')
	plt.title(f'SNR Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict)
	plt.show()
	plt.savefig(join(MEASURE_FOLDER, 'SNRplot.png'), dpi=200)
	plt.close()

	#---------------------------------------------------
	# Plotting 2 body decay rate
	#dfs = [df for df in groups.values()]

	for i, df  in enumerate(dfs[:]):
			data = df
			freqs = ((384182.5-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
			betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
			freqs = sorted(freqs)
			plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{groupbyKey}={data.iloc[10][groupbyKey]:.2f}")
	plt.legend()
	plt.xlabel(r'$\Delta$ (GHz)')
	plt.ylabel(r'$\beta_{\mathrm{eff}}$ ')
	plt.savefig(join(MEASURE_FOLDER, 'betaVsFreq.png'), dpi=200) 
	plt.title(f'2-body Decay Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict) 
	plt.show()
	plt.close()
	#*-----------------------
	#* MULTIPLE MEGARUN
	#*-----------------------
 
	folders = [os.path.join(EXP_FOLDER, 'MegaRuns', path ) for path in ['testPArunMega7', 'testPArunMega8']]

	dfs_mega = [get_data_frame(measure_folder, cache_all=True) for measure_folder in folders]

	groupbyKey = 'pump_reference'
	titleKey = 'pump_AOM_freq'

	dfs_grouped = [df_mega.groupby(by=groupbyKey) for df_mega in dfs_mega]
	min_ratios = [df_grouped['ratio'].min() for df_grouped in dfs_grouped]

	groupss = [dict(list(df_grouped)) for df_grouped in dfs_grouped]
	dfs = [ [df for df in groups.values()] for groups in groupss]
 
	for row in dfs:
		data = row[3]
	
		freqs = ((384182.5-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
		freqs = sorted(freqs)
		plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{titleKey}={data.iloc[10][titleKey]:.2f}")
		plt.title(f'{groupbyKey} = {data.iloc[10][groupbyKey]:.2f}')
	plt.legend()
	
	#*-----------------------
	#* FULL RUNS
	#*-----------------------
	
	MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'testPArunFull4')
	df = get_data_frame(MEASURE_FOLDER)
	df.dropna(inplace=True)

	dfc = df.copy()
	df_grouped = df.groupby(by=['pump_reference', 'pump_AOM_freq'])
	groups = dict(list(df_grouped))
	dfs = [df for df in groups.values()]
 
	max_freqs = [384182.8]*30
	zipped_data = list(zip(dfs, max_freqs))
	fig1, ax1s = plt.subplots(4)
	fig2, ax2s = plt.subplots(5)
	for i, (df, max_freq)  in enumerate(zipped_data[:]):
		j1 = i%4
		j2 = i//4
		
		data = df.dropna()
		freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-0.0)*FREQVSCURR)
		ax2s[j2] = plot_spline_fit(ax2s[j2], x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i%4}', mfc=f'C{i%4}',color=f'C{i%4}', s=0.0, ms=5, figsize=(5, 25), linewidth=1.5, label=f"Detuning  = { 180-2*df.iloc[10]['pump_AOM_freq'] :.2f}", fig=fig2)
		ax2s[j2].set_title(f"Pump Amplituide = { df.iloc[10]['pump_reference'] :.2f}", **titledict)
		ax2s[j2].legend()
		
		ax1s[j1] = plot_spline_fit(ax1s[j1], x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i//4}', mfc=f'C{i//4}',color=f'C{i//4}', s=0.0, ms=5, figsize=(5, 25), linewidth=1.5, label=f"Pump Amplitude = { df.iloc[10]['pump_reference'] :.2f}", fig=fig1)
		ax1s[j1].set_title(f"Detuning  = { 180-2*df.iloc[10]['pump_AOM_freq'] :.2f}", **titledict)
		ax1s[j1].legend()
		
	fig1.tight_layout()
	fig2.tight_layout()

	fig1.savefig(os.path.join(MEASURE_FOLDER, 'lossFeaturesDet.png'))
	plt.show()
	plt.close()

	fig2.savefig(os.path.join(MEASURE_FOLDER, 'lossFeaturesPA.png'))
	plt.show()
	plt.close()
	
	SNRdata = df_grouped['ratio'].max() - df_grouped['ratio'].min()
	SNRdf = SNRdata.reset_index()
	SNRdf.columns = ['pump_reference', 'pump_AOM_freq', 'SNR']
	pivot_table = SNRdf.pivot('pump_reference', 'pump_AOM_freq', 'SNR')
	xticklabels = [f'{180-2*x:.2f}' for x in pivot_table.columns]
	yticklabels = [f'{y:.2f}' for y in pivot_table.index]
	sns.heatmap(pivot_table, annot=True, fmt='.2f', xticklabels=xticklabels, yticklabels=yticklabels)
	plt.xlabel("Detuning (MHz)")
	plt.ylabel("Pump Reference")
	plt.grid()
	plt.savefig(os.path.join(MEASURE_FOLDER, 'heatmap.png'))
	plt.show()
	plt.close()

	# fig, ax = plt.subplots()
	# for i, (df, max_freq)  in enumerate(zipped_data[:]):
	# 	data = df.dropna()
	# 	freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		
	# 	ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.0, ms=5, figsize=(10, 10), linewidth=2.5)
		
	# 	plt.title(f"Pump Amplituide = { df.iloc[10]['pump_reference'] :.2f}, \
	# 				sDetuning  = { 180-2*df.iloc[10]['pump_AOM_freq'] :.2f}", **titledict)

	# 	plt.legend()

	# 	plt.savefig(os.path.join(MEASURE_FOLDER, f'lossFeatures{i}.png'))
	# 	plt.show()
		
	# 	plt.close()
	# 	fig, ax = plt.subplots()



def freq_misc():
	WDATA_FOLDER = r'C:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\CATMeasurements\CATcurrTestrun3.csv'
	freq_data = add_wavemeter_data('', WDATA_FOLDER)
	levels = staircase_fit(freq_data[0], peak_height=0.2, distance=50, data_offset=1, window_size=1) 

	plt.close()
 
	x = np.linspace(0, 4.9, 25)
	y = levels
 
	plt.plot(levels, '-o')
	plt.title('Levels plot')
	
	m, b, fit_line = my_linear_fit(x, y)
def save_fit_results(run_path, plot=False, bkfile=False, 
					 CATbaseline=True, MOTbaseline=True,
					 initRFit=True, loadFit=True,deloadFit=True,reloadFit=True,
					 storeFitResults=True):
	filename = os.path.join(run_path, 'data.csv')
	bkfilename = os.path.join(run_path, 'data_oldPD.csv')
	settingsname = os.path.join(run_path, 'Settings.txt')
	
	dataHost = MTdataHost(SAMPLE_RATE)
	dataHost.loadCATdata(fileName=filename, settingsName=settingsname)
	
	if bkfile:
		dataHost.CATbackgroundData(bkfilename)
	
	#dataHost.setAllCAT(0.002)
	if CATbaseline:
		dataHost.setCATbaseline(0.002)
	if MOTbaseline:
		dataHost.setBaseline(0.002)
	if loadFit:
		dataHost.setLoading(0.002)
	if initRFit and loadFit:
		dataHost.initFit, dataHost.initX = dataHost.setInitialLoad(0.002)
	if deloadFit:
		dataHost.setDeloading(0.002)
		dataHost.plotDeloadFit(run_path)# TODO: currently just stores the deloading times and voltages
	if reloadFit:
		dataHost.setReloadVolt(0.002)

	if CATbaseline and MOTbaseline and loadFit and reloadFit:
	# steady state ratio fraction
		dataHost.ratio = dataHost.reloadVolt / dataHost.motSS

		dataHost.ratioErr = dataHost.ratio * ((dataHost.reloadVoltErr/dataHost.reloadVolt)**2 + (dataHost.motSSErr/dataHost.motSS)**2)**(0.5)
	
		# if abs(dataHost.ratioErr / dataHost.ratio) > 0.1:
		#     dataHost.ratioErr = abs(0.015*dataHost.ratio)
		if dataHost.ratioErr < 0.001:
			dataHost.ratioErr = 0.001
		if dataHost.ratio < 0:
			dataHost.ratio = 0
			
		# TODO: this information is useless
	print('File loaded: RFmin = {} MHz, t_mt = {:.3f} s.'.format(dataHost.settings['fmin'], dataHost.settings['wait_mtrap']))
	
	resultDict = dataHost.getResults(run_path, store=storeFitResults)
 
	if plot:
		dataHost.storeFits(run_path, combined=True, separate=True)
  
	return resultDict, dataHost.settings
		
def get_timestamp(run_path):
	timestamp = datetime.strptime(os.path.split(run_path)[-1].split('_')[0], dateformat)
	return timestamp

def extract_fit(run_path, plot=True, cache_failed=True, cache_all=True):
	"""Gather relevant data from each measurement run

	Args:
		run_path : absolute path to the run directory
		plot (bool, optional): plot fits. Defaults to True.
		cache_failed (bool, optional): Cache failed fits. If false, refit. Doesn't refit non-failed fits. Defaults to True.
		cache_all (bool, optional): If false, ignore any cached fit_results. Defaults to True.

	Returns:
		a 3-tuple (fit_results, settings, timestamp)
	"""
	fit_results, settings, timestamp = {}, {}, None
	
	if not os.path.isdir(run_path):
		return fit_results, settings, timestamp  # directory is not a run directory

	try:
		timestamp = get_timestamp(run_path)
  
  	# TODO: specify which error to catch
	except Exception as e:
		print("Error extracting timestamp from: ", run_path)
		print(traceback.format_exc())
	
	MAT_fit_cache_path = os.path.join(run_path, 'resultDict.txt')
 
	if not os.path.exists(MAT_fit_cache_path) or not cache_all:
		
		try: 
			fit_results, settings = save_fit_results(run_path, plot=plot)
		except Exception as e:
			print(traceback.format_exc())
			print("Fitting ERROR at ", os.path.basename(run_path), '\n')

			with open(MAT_fit_cache_path, 'w') as f:
				f.write(str('MAT fit failed'))
		
	else:
		print("Accessing cached results from :", os.path.basename(run_path))
  
		fit_results = open(MAT_fit_cache_path, 'r').read()

		if fit_results == 'MAT fit failed':
			if not cache_failed:
				# fit regardless of cached result
				try: 
					fit_results, settings = save_fit_results(run_path, plot=plot)
				except Exception as e:
					print(traceback.format_exc())
					print("Fitting ERROR at ", os.path.basename(run_path), '\n')
	
					with open(MAT_fit_cache_path, 'w') as f:
						f.write(str('MAT fit failed'))	
			else:
				print("Failed fit at :", os.path.basename(run_path))
				fit_results = {}

		else:
			fit_results = eval(open(MAT_fit_cache_path, 'r').read())
   
			settingsname = os.path.join(run_path, 'Settings.txt')
			settings = eval(open(settingsname, 'r').read())
   
	return fit_results, settings, timestamp


def get_row(run_path, **kwargs):
			
   fit_results, settings, timestamp = extract_fit(run_path, **kwargs)
   row = {**fit_results, **settings, **{'timestamp':timestamp}}
   
   return row

   
def get_data_frame(data_dir, parallel=True, in_process_run=False, **kwargs):
   
	run_path_arr = []
	rows = []
	
	for relative_path in os.listdir(data_dir):
		run_path_arr.append(os.path.join(data_dir, relative_path))
  
	if in_process_run:
		run_path_arr.pop()

	run_path_arr = sorted(run_path_arr)
	
	if parallel:
		with Pool(4) as p:
			rows = list(tqdm(p.imap(partial(get_row, **kwargs), run_path_arr), total=len(run_path_arr)))
	
	else:
		for run_path in tqdm(run_path_arr):
		   rows.append(get_row(run_path, **kwargs))
	return pd.DataFrame.from_dict(rows)

def add_wavemeter_data(df, wmeter_csv_path, window_size=100, num_rows=50):


	"""Extract unique frequnecy values from wavemeter data
	Returns:
		unique_levels (list): unique frequency values in wavemeter data
	"""
 
	# TODO: modify dataframe in place with frequency data
 
	wdata = pd.read_csv(wmeter_csv_path, skiprows=2)
	wdata.dropna(inplace=True)
	freq_data = np.array(wdata.iloc[:, 0])
	try:
		freq_data = np.array([float(item) for item in freq_data if item.replace('.','').isdigit()])
	except Exception as e:
		print(e)
 
	max_freq = freq_data.max()
	min_freq = freq_data.min()

	return freq_data, max_freq, min_freq

def plot_results(ax, dfs, max_freq, min_freq=0.0, mfc='red', fmt='o', ms=5, save_folder=False, xscale=1.0, yscale=1.0, **kwargs):
	FREQVSVOLT = 221.0 
	FREQVSCURR = 1.13
	
	if not type(dfs) == list:
		freqs = ((max_freq-PUMP_FREQUENCY)-(dfs.dropna()['tempV']-dfs.dropna()['tempV'].min())*FREQVSVOLT- (dfs.dropna()['currV']-dfs.dropna()['currV'].min())*FREQVSCURR)*xscale
		dfs=[dfs]
  
	plt.gcf().set_dpi(300)
	
	for df in dfs:
		df = df.dropna()
		ax.errorbar(freqs, 
			   df['ratio']*yscale,
			   yerr=df['ratioErr'],
			   fmt=fmt, mfc=mfc, color='black', ms=ms, **kwargs)
		ax.set_ylabel(r'$\mathbf{\frac{V_{ss, cat}}{V_{ss}}} $ ', **labeldict)
		ax.set_xlabel(r'$\Delta $ (GHz)', **labeldict)
	
	if save_folder:
		plt.savefig(os.path.join(save_folder, 'ratio_vs_freq.png'))
	
	return freqs, ax
  
	#return plt.gca(), plt.gcf()
	#plt.show()

def plot_spline_fit(ax, x, y, s=1, yerr=None, color='black', scolor='black',figsize=(12,5), save_folder=None, title='',alpha=0.5,dpi=200, label='plot', fig=None,**kwargs):
	from scipy.interpolate import splev, splrep
	xnew = np.linspace(min(x), max(x), 3*len(x) )

	y = [b for a,b in sorted(zip(x,y), key=lambda pair: pair[0])]
	if yerr is not None:
		yerr = [b for a,b in sorted(zip(x,yerr), key=lambda pair: pair[0])]
 
	x = sorted(x)


	spl = splrep(x, y, s=s)
	ynew = splev(xnew, spl)

	if fig is None:
		plt.gcf().set_dpi(dpi)
		plt.gcf().set_size_inches(figsize)
	else:
		fig.set_dpi(dpi)
		fig.set_size_inches(figsize)
	if yerr is not None:
		ax.errorbar(x, y, yerr=yerr, fmt='o', **kwargs)
	else:
		ax.plot(x,y, 'o', **kwargs)
	ax.plot(xnew, ynew, '-', color=scolor, alpha=alpha, label=label, **kwargs)
 
	ax.set_ylabel(r'$\mathbf{\frac{V_{ss, cat}}{V_{ss}}} $ ', **labeldict)
	ax.set_xlabel(r'$\Delta $ (GHz)', **labeldict)
 
	ax.set_title(title, **titledict)
	if save_folder:
		plt.savefig(os.path.join(save_folder, 'spline_ratio_vs_freq.png'))
	
	return ax
 
def plot_polyfit(x_data, y_data, spline_degree):
	
	coefficients = np.polyfit(x_data, y_data, spline_degree)

	x_interp = np.linspace(min(x_data), max(x_data), 100)
	y_interp = np.polyval(coefficients, x_interp)

	plt.scatter(x_data, y_data, label='Original Data')
	plt.plot(x_interp, y_interp, label='Polynomial Interpolation (Degree={})'.format(spline_degree))

def collect_plots(source, destination, plot_name):
	print(f'Collecting plots from {os.path.basename(source)}')
	import shutil

	os.makedirs(destination, exist_ok=True)

	plot_files = []
	for root, dirs, files in os.walk(source):
		
		for file in files:
			if file == plot_name:
				plot_files.append(os.path.join(root, file))

	for i, plot_file in enumerate(plot_files, start=0):
		new_filename = f'{i}{plot_name}'
		destination_path = os.path.join(destination, new_filename)
		shutil.copy(plot_file, destination_path)

def create_GIF(images_folder, image_name):
	import imageio
	with imageio.get_writer(os.path.join(images_folder, f'{image_name}movie.gif'), mode='I', duration=0.5) as writer:
		for filename in os.listdir(images_folder):
			if image_name in filename:
				image = imageio.imread(os.path.join(images_folder, filename))
				writer.append_data(image)

def staircase_fit(data, peak_height=0.1, distance=100, data_offset=1, window_size=1, inc_final_peak=True):
	def moving_average(arr, window_size):
		weights = np.ones(window_size) / window_size
		return np.convolve(arr, weights, mode='valid')

	convdata1 = moving_average(data, window_size)
	convdata2 = moving_average(data[data_offset+1:], window_size )
	final = convdata1[:len(convdata2)]-convdata2
	# plt.plot(data)
	# plt.plot(convdata1)
	# plt.plot(convdata2)
	# plt.show()
	# plt.plot(final)

	
	from scipy.signal import find_peaks
	x= final
	peaks, _ = find_peaks(x, height=peak_height, distance=distance)
	peaks = np.insert(peaks, 0, 0)
	levels = []
	plot_arr = []
	for i, peak in enumerate(peaks):
		
		if i < len(peaks) - 1:
			temp = data[ peaks[i]:peaks[i+1] ]
			plot_arr.extend( np.ones_like(temp)*np.mean(temp))
			levels.append(np.mean(temp))
	if inc_final_peak:
		temp = data[peaks[-1]:]
		levels.append(np.mean(temp))
		plot_arr.extend( np.ones_like(temp)*np.mean(temp))
  
	plt.plot(x)
	plt.plot(peaks, x[peaks], "x")
	plt.plot(np.zeros_like(x), "--", color="gray")
	plt.show()
	
	plt.plot(data)
	plt.plot(np.ravel((plot_arr)))
	plt.show()
	plt.close()
	plt.plot(np.array(levels)[np.where(abs(np.diff(levels))>0.05)[0]], 'o', ms=5)
 
	return levels

def load_single_run(run_path):
	filename = os.path.join(run_path, 'data.csv')
	bkfilename = os.path.join(run_path, 'data_oldPD.csv')
	settingsname = os.path.join(run_path, 'Settings.txt')
	
	dh1 = MTdataHost(SAMPLE_RATE)
	dh1.loadCATdata(fileName=filename, settingsName=settingsname)
	
	return dh1
	
def load_mega_run(MEASURE_FOLDER, groupbyKey, titleKey, plot=True, save_plots=False, max_freq=384182.5, **kwargs):
    # TODO: use the plot flag to do something?
    
	df = get_data_frame(MEASURE_FOLDER,
						**kwargs)
	dfc= df.copy()
	#df.dropna(inplace=True)

	df_grouped = df.groupby(by=groupbyKey)
	min_ratios = df_grouped['ratio'].min()

	groups = dict(list(df_grouped))
	dfs = [df for df in groups.values()]

	# plotting ratio vs freq
	max_freqs = [max_freq]*len(dfs)
	zipped_data = list(zip(dfs, max_freqs))
	fig1, ax = plt.subplots()
	for i, (df, max_freq)  in enumerate(zipped_data[:]):
		data = df
		freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.0, ms=5, figsize=(10, 10), label=f"{groupbyKey} = { data.iloc[10][groupbyKey] :.2f}", linewidth=2.5)


	plt.legend()
	
	plt.title(f'Loss Features, {titleKey} = {data[titleKey].mean():.2f}', **titledict)
	plt.show()

	#---------------------------------------------------
	fig2 = plt.figure(2)
	x = [df[groupbyKey].mean() for df in dfs]
	y = [(df['ratio'].max() - df['ratio'].min()) for df in dfs]
	plt.plot( x, y ,'-o')
	plt.xlabel(groupbyKey)
	plt.ylabel(r'SNR $ = V_{ss, off} - V_{ss, on}$ ')
	plt.title(f'SNR Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict)
	plt.show()
	
	#---------------------------------------------------------
	fig3=plt.figure(3)
	for i, df  in enumerate(dfs[:]):
		data = df
		freqs = ((384182.5-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
		freqs = sorted(freqs)
		plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{groupbyKey}={data.iloc[10][groupbyKey]:.2f}")
	plt.legend()
	plt.xlabel(r'$\Delta$ (GHz)')
	plt.ylabel(r'$\beta_{\mathrm{eff}}$ ')
	plt.title(f'2-body Decay Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict) 
	plt.show()

	if save_plots:
		fig1.savefig(os.path.join(MEASURE_FOLDER, 'lossFeatures.png'))
		fig2.savefig(join(MEASURE_FOLDER, 'SNRplot.png'), dpi=200)
		fig3.savefig(join(MEASURE_FOLDER, 'betaVsFreq.png'), dpi=200) 
	return dfc


if __name__ == '__main__':
	# run_path = r"C:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\CATMeasurements\testPArun9\16-53-10"
	# filename = os.path.join(run_path, 'data.csv')
	# bkfilename = os.path.join(run_path, 'data_oldPD.csv')
	# settingsname = os.path.join(run_path, 'Settings.txt')
	
	# dh1 = MTdataHost(SAMPLE_RATE)
	# dh1.loadCATdata(fileName=filename, settingsName=settingsname)
	
	# dh1.setAllCAT(0.002)
 
 	#dh1.CATbackgroundData(bkfilename)
	
	pass


folders = [join(EXP_FOLDER, 'RepumpSizeTrapDepthStudy', path ) for path in ['PAsmallRepumpSizeTest', 'PAsmallRepumpSizeTest2', 'PAsmallRepumpSizeTest3', 'PAwideRepumpSizeTest', 'PAwideRepumpSizeTest2']]

dfs = [get_data_frame(measure_folder, cache_all=False, plot=False) for measure_folder in folders][1:]
labels = ['Narrow Beam' , 'Narrow Beam', 'Narrow Beam', 'Wide Beam', 'Wide Beam' ][1:]
max_freqs = [384182.6]*len(dfs)
zipped_data = list(zip(dfs, max_freqs))
for i, (df, max_freq)  in enumerate(zipped_data[:]):
    data=df
    freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
    betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
    freqs = sorted(freqs)
    plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{labels[i]}")
    plt.xlabel(r'$\Delta$ (GHz)')
    plt.ylabel(r'$\beta$')
    
plt.legend()

100%|██████████| 24/24 [00:21<00:00,  1.12it/s]
100%|██████████| 23/23 [00:18<00:00,  1.25it/s]
100%|██████████| 23/23 [00:15<00:00,  1.45it/s]
100%|██████████| 24/24 [00:15<00:00,  1.53it/s]
100%|██████████| 22/22 [00:14<00:00,  1.53it/s]

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\core\indexes\base.py:3802, in Index.get_loc(self, key, method, tolerance)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3800'>3801</a> try:
-> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3801'>3802</a>     return self._engine.get_loc(casted_key)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3802'>3803</a> except KeyError as err:

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\_libs\index.pyx:138, in pandas._libs.index.IndexEngine.get_loc()

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\_libs\index.pyx:165, in pandas._libs.index.IndexEngine.get_loc()

File pandas\_libs\hashtable_class_helper.pxi:5745, in pandas._libs.hashtable.PyObjectHashTable.get_item()

File pandas\_libs\hashtable_class_helper.pxi:5753, in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: 'tempV'

The above exception was the direct cause of the following exception:

KeyError                                  Traceback (most recent call last)
Cell In[2], line 9
      7 for i, (df, max_freq)  in enumerate(zipped_data[:]):
      8     data=df
----> 9     freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
     10     betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
     11     freqs = sorted(freqs)

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\core\frame.py:3807, in DataFrame.__getitem__(self, key)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3804'>3805</a> if self.columns.nlevels > 1:
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3805'>3806</a>     return self._getitem_multilevel(key)
-> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3806'>3807</a> indexer = self.columns.get_loc(key)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3807'>3808</a> if is_integer(indexer):
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3808'>3809</a>     indexer = [indexer]

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\core\indexes\base.py:3804, in Index.get_loc(self, key, method, tolerance)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3801'>3802</a>     return self._engine.get_loc(casted_key)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3802'>3803</a> except KeyError as err:
-> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3803'>3804</a>     raise KeyError(key) from err
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3804'>3805</a> except TypeError:
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3805'>3806</a>     # If we have a listlike key, _check_indexing_error will raise
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3806'>3807</a>     #  InvalidIndexError. Otherwise we fall through and re-raise
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3807'>3808</a>     #  the TypeError.
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3808'>3809</a>     self._check_indexing_error(key)

KeyError: 'tempV'


folders = [join(EXP_FOLDER, 'RepumpSizeTrapDepthStudy', path ) for path in ['PAsmallRepumpSizeTest', 'PAsmallRepumpSizeTest2', 'PAsmallRepumpSizeTest3', 'PAwideRepumpSizeTest', 'PAwideRepumpSizeTest2']]

dfs = [get_data_frame(measure_folder, cache_all=False, plot=False, parallel=False) for measure_folder in folders[1]]
labels = ['Narrow Beam' , 'Narrow Beam', 'Narrow Beam', 'Wide Beam', 'Wide Beam' ][1:]
max_freqs = [384182.6]*len(dfs)
zipped_data = list(zip(dfs, max_freqs))
for i, (df, max_freq)  in enumerate(zipped_data[:]):
    data=df
    freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
    betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
    freqs = sorted(freqs)
    plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{labels[i]}")
    plt.xlabel(r'$\Delta$ (GHz)')
    plt.ylabel(r'$\beta$')
    
plt.legend()

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
Cell In[3], line 3
      1 folders = [join(EXP_FOLDER, 'RepumpSizeTrapDepthStudy', path ) for path in ['PAsmallRepumpSizeTest', 'PAsmallRepumpSizeTest2', 'PAsmallRepumpSizeTest3', 'PAwideRepumpSizeTest', 'PAwideRepumpSizeTest2']]
----> 3 dfs = [get_data_frame(measure_folder, cache_all=False, plot=False, parallel=False) for measure_folder in folders[1]]
      4 labels = ['Narrow Beam' , 'Narrow Beam', 'Narrow Beam', 'Wide Beam', 'Wide Beam' ][1:]
      5 max_freqs = [384182.6]*len(dfs)

Cell In[3], line 3, in <listcomp>(.0)
      1 folders = [join(EXP_FOLDER, 'RepumpSizeTrapDepthStudy', path ) for path in ['PAsmallRepumpSizeTest', 'PAsmallRepumpSizeTest2', 'PAsmallRepumpSizeTest3', 'PAwideRepumpSizeTest', 'PAwideRepumpSizeTest2']]
----> 3 dfs = [get_data_frame(measure_folder, cache_all=False, plot=False, parallel=False) for measure_folder in folders[1]]
      4 labels = ['Narrow Beam' , 'Narrow Beam', 'Narrow Beam', 'Wide Beam', 'Wide Beam' ][1:]
      5 max_freqs = [384182.6]*len(dfs)

c:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\analysis\analysis_test.py in line 423, in get_data_frame(data_dir, parallel, in_process_run, **kwargs)
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=419'>420</a> run_path_arr = []
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=420'>421</a> rows = []
--> <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=422'>423</a> for relative_path in os.listdir(data_dir):
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=423'>424</a> 	run_path_arr.append(os.path.join(data_dir, relative_path))
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=425'>426</a> if in_process_run:

FileNotFoundError: [WinError 3] The system cannot find the path specified: 'C'


folders = [join(EXP_FOLDER, 'RepumpSizeTrapDepthStudy', path ) for path in ['PAsmallRepumpSizeTest', 'PAsmallRepumpSizeTest2', 'PAsmallRepumpSizeTest3', 'PAwideRepumpSizeTest', 'PAwideRepumpSizeTest2']]

dfs = [get_data_frame(measure_folder, cache_all=False, plot=False, parallel=False) for measure_folder in [folders[1]]]
labels = ['Narrow Beam' , 'Narrow Beam', 'Narrow Beam', 'Wide Beam', 'Wide Beam' ][1:]
max_freqs = [384182.6]*len(dfs)
zipped_data = list(zip(dfs, max_freqs))
for i, (df, max_freq)  in enumerate(zipped_data[:]):
    data=df
    freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
    betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
    freqs = sorted(freqs)
    plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{labels[i]}")
    plt.xlabel(r'$\Delta$ (GHz)')
    plt.ylabel(r'$\beta$')
    
plt.legend()

  4%|▍         | 1/23 [00:01<00:33,  1.51s/it]

Traceback (most recent call last):
  File "<ipython-input-1-dd11578c6459>", line 373, in extract_fit
    fit_results, settings = save_fit_results(run_path, plot=plot)
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<ipython-input-1-dd11578c6459>", line 312, in save_fit_results
    dataHost.plotDeloadFit(run_path)# TODO: currently just stores the deloading times and voltages
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\analysis\MT_class_PID_new.py", line 741, in plotDeloadFit
    plt.plot(self.deloadTime, self.deloadFit, 'r-', label='best fit')
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\pyplot.py", line 2812, in plot
    return gca().plot(
           ^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_axes.py", line 1688, in plot
    lines = [*self._get_lines(*args, data=data, **kwargs)]
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_base.py", line 311, in __call__
    yield from self._plot_args(
               ^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_base.py", line 504, in _plot_args
    raise ValueError(f"x and y must have same first dimension, but "
ValueError: x and y must have same first dimension, but have shapes (59840,) and (10000,)

Fitting ERROR at  16-14-11

  9%|▊         | 2/23 [00:03<00:36,  1.73s/it]

Traceback (most recent call last):
  File "<ipython-input-1-dd11578c6459>", line 373, in extract_fit
    fit_results, settings = save_fit_results(run_path, plot=plot)
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<ipython-input-1-dd11578c6459>", line 312, in save_fit_results
    dataHost.plotDeloadFit(run_path)# TODO: currently just stores the deloading times and voltages
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\analysis\MT_class_PID_new.py", line 741, in plotDeloadFit
    plt.plot(self.deloadTime, self.deloadFit, 'r-', label='best fit')
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\pyplot.py", line 2812, in plot
    return gca().plot(
           ^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_axes.py", line 1688, in plot
    lines = [*self._get_lines(*args, data=data, **kwargs)]
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_base.py", line 311, in __call__
    yield from self._plot_args(
               ^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_base.py", line 504, in _plot_args
    raise ValueError(f"x and y must have same first dimension, but "
ValueError: x and y must have same first dimension, but have shapes (59840,) and (10000,)

Fitting ERROR at  16-15-42

 13%|█▎        | 3/23 [00:04<00:32,  1.64s/it]

Traceback (most recent call last):
  File "<ipython-input-1-dd11578c6459>", line 373, in extract_fit
    fit_results, settings = save_fit_results(run_path, plot=plot)
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<ipython-input-1-dd11578c6459>", line 312, in save_fit_results
    dataHost.plotDeloadFit(run_path)# TODO: currently just stores the deloading times and voltages
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\analysis\MT_class_PID_new.py", line 741, in plotDeloadFit
    plt.plot(self.deloadTime, self.deloadFit, 'r-', label='best fit')
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\pyplot.py", line 2812, in plot
    return gca().plot(
           ^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_axes.py", line 1688, in plot
    lines = [*self._get_lines(*args, data=data, **kwargs)]
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_base.py", line 311, in __call__
    yield from self._plot_args(
               ^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_base.py", line 504, in _plot_args
    raise ValueError(f"x and y must have same first dimension, but "
ValueError: x and y must have same first dimension, but have shapes (59840,) and (10000,)

Fitting ERROR at  16-17-16

 17%|█▋        | 4/23 [00:06<00:28,  1.50s/it]

Traceback (most recent call last):
  File "<ipython-input-1-dd11578c6459>", line 373, in extract_fit
    fit_results, settings = save_fit_results(run_path, plot=plot)
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<ipython-input-1-dd11578c6459>", line 312, in save_fit_results
    dataHost.plotDeloadFit(run_path)# TODO: currently just stores the deloading times and voltages
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\analysis\MT_class_PID_new.py", line 741, in plotDeloadFit
    plt.plot(self.deloadTime, self.deloadFit, 'r-', label='best fit')
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\pyplot.py", line 2812, in plot
    return gca().plot(
           ^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_axes.py", line 1688, in plot
    lines = [*self._get_lines(*args, data=data, **kwargs)]
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_base.py", line 311, in __call__
    yield from self._plot_args(
               ^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_base.py", line 504, in _plot_args
    raise ValueError(f"x and y must have same first dimension, but "
ValueError: x and y must have same first dimension, but have shapes (59840,) and (10000,)

Fitting ERROR at  16-18-51

 22%|██▏       | 5/23 [00:07<00:25,  1.39s/it]

Traceback (most recent call last):
  File "<ipython-input-1-dd11578c6459>", line 373, in extract_fit
    fit_results, settings = save_fit_results(run_path, plot=plot)
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<ipython-input-1-dd11578c6459>", line 312, in save_fit_results
    dataHost.plotDeloadFit(run_path)# TODO: currently just stores the deloading times and voltages
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\analysis\MT_class_PID_new.py", line 741, in plotDeloadFit
    plt.plot(self.deloadTime, self.deloadFit, 'r-', label='best fit')
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\pyplot.py", line 2812, in plot
    return gca().plot(
           ^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_axes.py", line 1688, in plot
    lines = [*self._get_lines(*args, data=data, **kwargs)]
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_base.py", line 311, in __call__
    yield from self._plot_args(
               ^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_base.py", line 504, in _plot_args
    raise ValueError(f"x and y must have same first dimension, but "
ValueError: x and y must have same first dimension, but have shapes (59840,) and (10000,)

Fitting ERROR at  16-20-26


folders = [join(EXP_FOLDER, 'RepumpSizeTrapDepthStudy', path ) for path in ['PAsmallRepumpSizeTest', 'PAsmallRepumpSizeTest2', 'PAsmallRepumpSizeTest3', 'PAwideRepumpSizeTest', 'PAwideRepumpSizeTest2']]

dfs = [get_data_frame(measure_folder, cache_all=False, plot=False, parallel=False) for measure_folder in [folders[1]]]
labels = ['Narrow Beam' , 'Narrow Beam', 'Narrow Beam', 'Wide Beam', 'Wide Beam' ][1:]
max_freqs = [384182.6]*len(dfs)
zipped_data = list(zip(dfs, max_freqs))
for i, (df, max_freq)  in enumerate(zipped_data[:]):
    data=df
    freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
    betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
    freqs = sorted(freqs)
    plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{labels[i]}")
    plt.xlabel(r'$\Delta$ (GHz)')
    plt.ylabel(r'$\beta$')
    
plt.legend()

  4%|▍         | 1/23 [00:01<00:24,  1.09s/it]

Traceback (most recent call last):
  File "<ipython-input-1-dd11578c6459>", line 373, in extract_fit
    fit_results, settings = save_fit_results(run_path, plot=plot)
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<ipython-input-1-dd11578c6459>", line 312, in save_fit_results
    dataHost.plotDeloadFit(run_path)# TODO: currently just stores the deloading times and voltages
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\analysis\MT_class_PID_new.py", line 741, in plotDeloadFit
    plt.plot(self.deloadTime[:int(self.deloadFitTime*self.sampleRate)], self.deloadFit, 'r-', label='best fit')
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\pyplot.py", line 2812, in plot
    return gca().plot(
           ^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_axes.py", line 1688, in plot
    lines = [*self._get_lines(*args, data=data, **kwargs)]
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_base.py", line 311, in __call__
    yield from self._plot_args(
               ^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_base.py", line 504, in _plot_args
    raise ValueError(f"x and y must have same first dimension, but "
ValueError: x and y must have same first dimension, but have shapes (59840,) and (10000,)

Fitting ERROR at  16-14-11

  9%|▊         | 2/23 [00:02<00:21,  1.03s/it]

Traceback (most recent call last):
  File "<ipython-input-1-dd11578c6459>", line 373, in extract_fit
    fit_results, settings = save_fit_results(run_path, plot=plot)
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<ipython-input-1-dd11578c6459>", line 312, in save_fit_results
    dataHost.plotDeloadFit(run_path)# TODO: currently just stores the deloading times and voltages
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\analysis\MT_class_PID_new.py", line 741, in plotDeloadFit
    plt.plot(self.deloadTime[:int(self.deloadFitTime*self.sampleRate)], self.deloadFit, 'r-', label='best fit')
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\pyplot.py", line 2812, in plot
    return gca().plot(
           ^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_axes.py", line 1688, in plot
    lines = [*self._get_lines(*args, data=data, **kwargs)]
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_base.py", line 311, in __call__
    yield from self._plot_args(
               ^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_base.py", line 504, in _plot_args
    raise ValueError(f"x and y must have same first dimension, but "
ValueError: x and y must have same first dimension, but have shapes (59840,) and (10000,)

Fitting ERROR at  16-15-42

 13%|█▎        | 3/23 [00:03<00:20,  1.02s/it]

Traceback (most recent call last):
  File "<ipython-input-1-dd11578c6459>", line 373, in extract_fit
    fit_results, settings = save_fit_results(run_path, plot=plot)
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<ipython-input-1-dd11578c6459>", line 312, in save_fit_results
    dataHost.plotDeloadFit(run_path)# TODO: currently just stores the deloading times and voltages
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\analysis\MT_class_PID_new.py", line 741, in plotDeloadFit
    plt.plot(self.deloadTime[:int(self.deloadFitTime*self.sampleRate)], self.deloadFit, 'r-', label='best fit')
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\pyplot.py", line 2812, in plot
    return gca().plot(
           ^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_axes.py", line 1688, in plot
    lines = [*self._get_lines(*args, data=data, **kwargs)]
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_base.py", line 311, in __call__
    yield from self._plot_args(
               ^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_base.py", line 504, in _plot_args
    raise ValueError(f"x and y must have same first dimension, but "
ValueError: x and y must have same first dimension, but have shapes (59840,) and (10000,)

Fitting ERROR at  16-17-16

 17%|█▋        | 4/23 [00:03<00:18,  1.05it/s]

Traceback (most recent call last):
  File "<ipython-input-1-dd11578c6459>", line 373, in extract_fit
    fit_results, settings = save_fit_results(run_path, plot=plot)
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<ipython-input-1-dd11578c6459>", line 312, in save_fit_results
    dataHost.plotDeloadFit(run_path)# TODO: currently just stores the deloading times and voltages
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\analysis\MT_class_PID_new.py", line 741, in plotDeloadFit
    plt.plot(self.deloadTime[:int(self.deloadFitTime*self.sampleRate)], self.deloadFit, 'r-', label='best fit')
    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\pyplot.py", line 2812, in plot
    return gca().plot(
           ^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_axes.py", line 1688, in plot
    lines = [*self._get_lines(*args, data=data, **kwargs)]
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_base.py", line 311, in __call__
    yield from self._plot_args(
               ^^^^^^^^^^^^^^^^
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\matplotlib\axes\_base.py", line 504, in _plot_args
    raise ValueError(f"x and y must have same first dimension, but "
ValueError: x and y must have same first dimension, but have shapes (59840,) and (10000,)

Fitting ERROR at  16-18-51

 17%|█▋        | 4/23 [00:04<00:22,  1.19s/it]

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
Cell In[5], line 3
      1 folders = [join(EXP_FOLDER, 'RepumpSizeTrapDepthStudy', path ) for path in ['PAsmallRepumpSizeTest', 'PAsmallRepumpSizeTest2', 'PAsmallRepumpSizeTest3', 'PAwideRepumpSizeTest', 'PAwideRepumpSizeTest2']]
----> 3 dfs = [get_data_frame(measure_folder, cache_all=False, plot=False, parallel=False) for measure_folder in [folders[1]]]
      4 labels = ['Narrow Beam' , 'Narrow Beam', 'Narrow Beam', 'Wide Beam', 'Wide Beam' ][1:]
      5 max_freqs = [384182.6]*len(dfs)

Cell In[5], line 3, in <listcomp>(.0)
      1 folders = [join(EXP_FOLDER, 'RepumpSizeTrapDepthStudy', path ) for path in ['PAsmallRepumpSizeTest', 'PAsmallRepumpSizeTest2', 'PAsmallRepumpSizeTest3', 'PAwideRepumpSizeTest', 'PAwideRepumpSizeTest2']]
----> 3 dfs = [get_data_frame(measure_folder, cache_all=False, plot=False, parallel=False) for measure_folder in [folders[1]]]
      4 labels = ['Narrow Beam' , 'Narrow Beam', 'Narrow Beam', 'Wide Beam', 'Wide Beam' ][1:]
      5 max_freqs = [384182.6]*len(dfs)

c:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\analysis\analysis_test.py in line 437, in get_data_frame(data_dir, parallel, in_process_run, **kwargs)
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=434'>435</a> else:
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=435'>436</a> 	for run_path in tqdm(run_path_arr):
--> <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=436'>437</a> 	   rows.append(get_row(run_path, **kwargs))
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=437'>438</a> return pd.DataFrame.from_dict(rows)

c:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\analysis\analysis_test.py in line 412, in get_row(run_path, **kwargs)
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=409'>410</a> def get_row(run_path, **kwargs):
--> <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=411'>412</a>    fit_results, settings, timestamp = extract_fit(run_path, **kwargs)
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=412'>413</a>    row = {**fit_results, **settings, **{'timestamp':timestamp}}
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=414'>415</a>    return row

c:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\analysis\analysis_test.py in line 373, in extract_fit(run_path, plot, cache_failed, cache_all)
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=369'>370</a> if not os.path.exists(MAT_fit_cache_path) or not cache_all:
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=371'>372</a> 	try: 
--> <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=372'>373</a> 		fit_results, settings = save_fit_results(run_path, plot=plot)
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=373'>374</a> 	except Exception as e:
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=374'>375</a> 		print(traceback.format_exc())

c:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\analysis\analysis_test.py in line 311, in save_fit_results(run_path, plot, bkfile, CATbaseline, MOTbaseline, initRFit, loadFit, deloadFit, reloadFit, storeFitResults)
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=308'>309</a> 	dataHost.initFit, dataHost.initX = dataHost.setInitialLoad(0.002)
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=309'>310</a> if deloadFit:
--> <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=310'>311</a> 	dataHost.setDeloading(0.002)
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=311'>312</a> 	dataHost.plotDeloadFit(run_path)# TODO: currently just stores the deloading times and voltages
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=312'>313</a> if reloadFit:

File c:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\analysis\MT_class_PID_new.py:734, in MTdataHost.setDeloading(self, timeBuffer)
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/MT_class_PID_new.py?line=730'>731</a> x = sorted(x)
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/MT_class_PID_new.py?line=732'>733</a> model = Model(N2)
--> <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/MT_class_PID_new.py?line=733'>734</a> result = model.fit(y, t=x, params=params2)
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/MT_class_PID_new.py?line=735'>736</a> self.deloadFit = result.best_fit
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/MT_class_PID_new.py?line=736'>737</a> self.betaPA, self.betaPAErr = result.params['beta'].value , result.params['beta'].stderr

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\lmfit\model.py:1103, in Model.fit(self, data, params, weights, method, iter_cb, scale_covar, verbose, fit_kws, nan_policy, calc_covar, max_nfev, coerce_farray, **kwargs)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/model.py?line=1096'>1097</a>     fit_kws = {}
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/model.py?line=1098'>1099</a> output = ModelResult(self, params, method=method, iter_cb=iter_cb,
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/model.py?line=1099'>1100</a>                      scale_covar=scale_covar, fcn_kws=kwargs,
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/model.py?line=1100'>1101</a>                      nan_policy=self.nan_policy, calc_covar=calc_covar,
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/model.py?line=1101'>1102</a>                      max_nfev=max_nfev, **fit_kws)
-> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/model.py?line=1102'>1103</a> output.fit(data=data, weights=weights)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/model.py?line=1103'>1104</a> output.components = self.components
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/model.py?line=1104'>1105</a> return output

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\lmfit\model.py:1467, in ModelResult.fit(self, data, params, weights, method, nan_policy, **kwargs)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/model.py?line=1464'>1465</a> self.userkws.update(kwargs)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/model.py?line=1465'>1466</a> self.init_fit = self.model.eval(params=self.params, **self.userkws)
-> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/model.py?line=1466'>1467</a> _ret = self.minimize(method=self.method)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/model.py?line=1467'>1468</a> self.model.post_fit(_ret)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/model.py?line=1468'>1469</a> _ret.params.create_uvars(covar=_ret.covar)

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\lmfit\minimizer.py:2345, in Minimizer.minimize(self, method, params, **kws)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/minimizer.py?line=2341'>2342</a>         if (key.lower().startswith(user_method) or
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/minimizer.py?line=2342'>2343</a>                 val.lower().startswith(user_method)):
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/minimizer.py?line=2343'>2344</a>             kwargs['method'] = val
-> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/minimizer.py?line=2344'>2345</a> return function(**kwargs)

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\lmfit\minimizer.py:1651, in Minimizer.leastsq(self, params, max_nfev, **kws)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/minimizer.py?line=1648'>1649</a> result.call_kws = lskws
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/minimizer.py?line=1649'>1650</a> try:
-> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/minimizer.py?line=1650'>1651</a>     lsout = scipy_leastsq(self.__residual, variables, **lskws)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/minimizer.py?line=1651'>1652</a> except AbortFitException:
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/minimizer.py?line=1652'>1653</a>     pass

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\scipy\optimize\_minpack_py.py:426, in leastsq(func, x0, args, Dfun, full_output, col_deriv, ftol, xtol, gtol, maxfev, epsfcn, factor, diag)
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/optimize/_minpack_py.py?line=423'>424</a>     if maxfev == 0:
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/optimize/_minpack_py.py?line=424'>425</a>         maxfev = 200*(n + 1)
--> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/optimize/_minpack_py.py?line=425'>426</a>     retval = _minpack._lmdif(func, x0, args, full_output, ftol, xtol,
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/optimize/_minpack_py.py?line=426'>427</a>                              gtol, maxfev, epsfcn, factor, diag)
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/optimize/_minpack_py.py?line=427'>428</a> else:
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/optimize/_minpack_py.py?line=428'>429</a>     if col_deriv:

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\lmfit\minimizer.py:548, in Minimizer.__residual(self, fvars, apply_bounds_transformation)
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/minimizer.py?line=544'>545</a>     self.result.success = False
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/minimizer.py?line=545'>546</a>     raise AbortFitException(f"fit aborted: too many function evaluations {self.max_nfev}")
--> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/minimizer.py?line=547'>548</a> out = self.userfcn(params, *self.userargs, **self.userkws)
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/minimizer.py?line=549'>550</a> if callable(self.iter_cb):
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/minimizer.py?line=550'>551</a>     abort = self.iter_cb(params, self.result.nfev, out,
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/lmfit/minimizer.py?line=551'>552</a>                          *self.userargs, **self.userkws)

KeyboardInterrupt:


import os
from os.path import join 
import sys
from functools import partial
sys.path.append(os.path.join(os.getcwd(), '..')) #adds directory below as valid path
from  datetime import datetime, timedelta
dateformat = "%H-%M-%S"
from collections import deque
import traceback
from multiprocessing import Pool
from tqdm import tqdm

import scipy.constants as spc
from lmfit import Model, create_params

from scipy.integrate import odeint
from scipy.optimize import curve_fit

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

plt.rcParams['axes.grid'] = True
plt.rcParams['grid.linestyle'] = '--'

from MT_class_PID_new import MTdataHost
from global_folder.myplotsty import *
from global_folder.my_helpers import *

PUMP_FREQUENCY = 384228.6
REPUMP_FREQUENCY = 384228.6 + 6.56
SAMPLE_RATE = 2000
FREQVSVOLT = 221.0 
FREQVSCURR = 1.13

# TODO: find a better place for this
EXP_FOLDER =r'C:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\CATMeasurements'
MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'testPArun16')
WDATA_FOLDER =os.path.join(MEASURE_FOLDER, 'testPArun16.csv')

# TODO: maybe make a run analysis class out of this?
def dump():
	fig, ax = plt.subplots()
	plot_results(ax, df, max_freq= 384219., fmt='o', mfc='red', save_folder=MEASURE_FOLDER)
	plt.savefig(os.path.join(MEASURE_FOLDER, 'ratio_vs_freq.png'))

	
	collect_plots(MEASURE_FOLDER, os.path.join(MEASURE_FOLDER, 'collected_plots'), 'deloadPhase.png')
 
	#*-----------------------
	#* SINGLE RUN
 	#*----------------------- 
  
	MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'testPArun9')
	df = get_data_frame(MEASURE_FOLDER)
	df.dropna(inplace=True)
	#freqs = plot_results(df, 384201., save_folder=MEASURE_FOLDER)
 
	data = df.dropna()
	freqs = ((384201-PUMP_FREQUENCY)-(data['tempV']-df['tempV'].min())*FREQVSVOLT- (data['currV']-df['currV'].min())*FREQVSCURR)
	fig, ax = plt.subplots()
	plot_spline_fit(ax=ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'], 
				 s=0.1, save_folder=MEASURE_FOLDER, 
				 mfc='red', color='black', 
				 title='Trap Depth = 1.99 K')
 
	# *-----------------------
	# * MULTIPLE RUN COMPARISON
 	# *-----------------------
	folders = [os.path.join(EXP_FOLDER, path ) for path in ['PArunHalfVarDet1', 'testPAVaryingCATampl']]
	dfs = [get_data_frame(measure_folder) for measure_folder in folders]

	dfs = [get_data_frame(measure_folder) for measure_folder in folders]

	max_freqs = [384182.6]*len(dfs)
	zipped_data = list(zip(dfs, max_freqs))
	for i, (df, max_freq)  in enumerate(zipped_data[:]):
		fig, ax = plt.subplots()
		data = df.dropna()
		freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'], scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.00, ms=5, save_folder=join(folders[i]))

	#*-----------------------
	#* PARSING WAVEMETER DATA
 	#*-----------------------

	MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'testPArun14')
	WDATA_FOLDER =os.path.join(MEASURE_FOLDER, 'testPArun14.csv')
	freq_data, max_freq, min_freq = add_wavemeter_data('', WDATA_FOLDER)
	data = freq_data[:]
	levels = staircase_fit(data)

	data = get_data_frame(MEASURE_FOLDER)
	data.dropna(inplace=True)
	freqs = ((max_freq)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
	plt.plot(freqs)
 
	#*-----------------------
	#* GETTING DEPTH_RATIO DATAFRAME
 	#*-----------------------
	MEASURE_FOLDER = r'C:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\CATMeasurements\relScatRate'
	depth_ratios_df = get_rel_scattering_df(MEASURE_FOLDER)
	df_slice = depth_ratios_df[(depth_ratios_df['pa1']==1.85) & (depth_ratios_df['pd1']==84) & (depth_ratios_df['pd2'] == 84)]
	x = df_slice['pa2']
	y = df_slice['depth_ratio']
	plt.plot(x,y, 'o')

	#*-----------------------
	#* MEGA_RUN
 	#*-----------------------	
	MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'MegaRuns', 'testPArunMega3')
	df = get_data_frame(MEASURE_FOLDER,
						plot=False,
						cache_all=True)
	dfc= df.copy()
	#df.dropna(inplace=True)

	groupbyKey = 'pump_reference'
	titleKey = 'pump_AOM_freq'

	df_grouped = df.groupby(by=groupbyKey)
	min_ratios = df_grouped['ratio'].min()

	groups = dict(list(df_grouped))
	dfs = [df for df in groups.values()]

	# plotting ratio vs freq
	max_freqs = [384182.5]*len(dfs)
	zipped_data = list(zip(dfs, max_freqs))
	fig, ax = plt.subplots()
	for i, (df, max_freq)  in enumerate(zipped_data[:]):
		data = df
		freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.0, ms=5, figsize=(10, 10), label=f"{groupbyKey} = { data.iloc[10][groupbyKey] :.2f}", linewidth=2.5)

	plt.legend()
	plt.savefig(os.path.join(MEASURE_FOLDER, 'lossFeatures.png'))
	plt.title(f'Loss Features, {titleKey} = {data[titleKey].mean():.2f}', **titledict)
	plt.show()
	plt.close()
	#---------------------------------------------------
	x = [df[groupbyKey].mean() for df in dfs]
	y = [(df['ratio'].max() - df['ratio'].min()) for df in dfs]
	plt.plot( x, y ,'-o')
	plt.xlabel(groupbyKey)
	plt.ylabel(r'SNR $ = V_{ss, off} - V_{ss, on}$ ')
	plt.title(f'SNR Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict)
	plt.show()
	plt.savefig(join(MEASURE_FOLDER, 'SNRplot.png'), dpi=200)
	plt.close()

	#---------------------------------------------------
	# Plotting 2 body decay rate
	#dfs = [df for df in groups.values()]

	for i, df  in enumerate(dfs[:]):
			data = df
			freqs = ((384182.5-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
			betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
			freqs = sorted(freqs)
			plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{groupbyKey}={data.iloc[10][groupbyKey]:.2f}")
	plt.legend()
	plt.xlabel(r'$\Delta$ (GHz)')
	plt.ylabel(r'$\beta_{\mathrm{eff}}$ ')
	plt.savefig(join(MEASURE_FOLDER, 'betaVsFreq.png'), dpi=200) 
	plt.title(f'2-body Decay Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict) 
	plt.show()
	plt.close()
	#*-----------------------
	#* MULTIPLE MEGARUN
	#*-----------------------
 
	folders = [os.path.join(EXP_FOLDER, 'MegaRuns', path ) for path in ['testPArunMega7', 'testPArunMega8']]

	dfs_mega = [get_data_frame(measure_folder, cache_all=True) for measure_folder in folders]

	groupbyKey = 'pump_reference'
	titleKey = 'pump_AOM_freq'

	dfs_grouped = [df_mega.groupby(by=groupbyKey) for df_mega in dfs_mega]
	min_ratios = [df_grouped['ratio'].min() for df_grouped in dfs_grouped]

	groupss = [dict(list(df_grouped)) for df_grouped in dfs_grouped]
	dfs = [ [df for df in groups.values()] for groups in groupss]
 
	for row in dfs:
		data = row[3]
	
		freqs = ((384182.5-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
		freqs = sorted(freqs)
		plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{titleKey}={data.iloc[10][titleKey]:.2f}")
		plt.title(f'{groupbyKey} = {data.iloc[10][groupbyKey]:.2f}')
	plt.legend()
	
	#*-----------------------
	#* FULL RUNS
	#*-----------------------
	
	MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'testPArunFull4')
	df = get_data_frame(MEASURE_FOLDER)
	df.dropna(inplace=True)

	dfc = df.copy()
	df_grouped = df.groupby(by=['pump_reference', 'pump_AOM_freq'])
	groups = dict(list(df_grouped))
	dfs = [df for df in groups.values()]
 
	max_freqs = [384182.8]*30
	zipped_data = list(zip(dfs, max_freqs))
	fig1, ax1s = plt.subplots(4)
	fig2, ax2s = plt.subplots(5)
	for i, (df, max_freq)  in enumerate(zipped_data[:]):
		j1 = i%4
		j2 = i//4
		
		data = df.dropna()
		freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-0.0)*FREQVSCURR)
		ax2s[j2] = plot_spline_fit(ax2s[j2], x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i%4}', mfc=f'C{i%4}',color=f'C{i%4}', s=0.0, ms=5, figsize=(5, 25), linewidth=1.5, label=f"Detuning  = { 180-2*df.iloc[10]['pump_AOM_freq'] :.2f}", fig=fig2)
		ax2s[j2].set_title(f"Pump Amplituide = { df.iloc[10]['pump_reference'] :.2f}", **titledict)
		ax2s[j2].legend()
		
		ax1s[j1] = plot_spline_fit(ax1s[j1], x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i//4}', mfc=f'C{i//4}',color=f'C{i//4}', s=0.0, ms=5, figsize=(5, 25), linewidth=1.5, label=f"Pump Amplitude = { df.iloc[10]['pump_reference'] :.2f}", fig=fig1)
		ax1s[j1].set_title(f"Detuning  = { 180-2*df.iloc[10]['pump_AOM_freq'] :.2f}", **titledict)
		ax1s[j1].legend()
		
	fig1.tight_layout()
	fig2.tight_layout()

	fig1.savefig(os.path.join(MEASURE_FOLDER, 'lossFeaturesDet.png'))
	plt.show()
	plt.close()

	fig2.savefig(os.path.join(MEASURE_FOLDER, 'lossFeaturesPA.png'))
	plt.show()
	plt.close()
	
	SNRdata = df_grouped['ratio'].max() - df_grouped['ratio'].min()
	SNRdf = SNRdata.reset_index()
	SNRdf.columns = ['pump_reference', 'pump_AOM_freq', 'SNR']
	pivot_table = SNRdf.pivot('pump_reference', 'pump_AOM_freq', 'SNR')
	xticklabels = [f'{180-2*x:.2f}' for x in pivot_table.columns]
	yticklabels = [f'{y:.2f}' for y in pivot_table.index]
	sns.heatmap(pivot_table, annot=True, fmt='.2f', xticklabels=xticklabels, yticklabels=yticklabels)
	plt.xlabel("Detuning (MHz)")
	plt.ylabel("Pump Reference")
	plt.grid()
	plt.savefig(os.path.join(MEASURE_FOLDER, 'heatmap.png'))
	plt.show()
	plt.close()

	# fig, ax = plt.subplots()
	# for i, (df, max_freq)  in enumerate(zipped_data[:]):
	# 	data = df.dropna()
	# 	freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		
	# 	ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.0, ms=5, figsize=(10, 10), linewidth=2.5)
		
	# 	plt.title(f"Pump Amplituide = { df.iloc[10]['pump_reference'] :.2f}, \
	# 				sDetuning  = { 180-2*df.iloc[10]['pump_AOM_freq'] :.2f}", **titledict)

	# 	plt.legend()

	# 	plt.savefig(os.path.join(MEASURE_FOLDER, f'lossFeatures{i}.png'))
	# 	plt.show()
		
	# 	plt.close()
	# 	fig, ax = plt.subplots()



def freq_misc():
	WDATA_FOLDER = r'C:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\CATMeasurements\CATcurrTestrun3.csv'
	freq_data = add_wavemeter_data('', WDATA_FOLDER)
	levels = staircase_fit(freq_data[0], peak_height=0.2, distance=50, data_offset=1, window_size=1) 

	plt.close()
 
	x = np.linspace(0, 4.9, 25)
	y = levels
 
	plt.plot(levels, '-o')
	plt.title('Levels plot')
	
	m, b, fit_line = my_linear_fit(x, y)
def save_fit_results(run_path, plot=False, bkfile=False, 
					 CATbaseline=True, MOTbaseline=True,
					 initRFit=True, loadFit=True,deloadFit=True,reloadFit=True,
					 storeFitResults=True):
	filename = os.path.join(run_path, 'data.csv')
	bkfilename = os.path.join(run_path, 'data_oldPD.csv')
	settingsname = os.path.join(run_path, 'Settings.txt')
	
	dataHost = MTdataHost(SAMPLE_RATE)
	dataHost.loadCATdata(fileName=filename, settingsName=settingsname)
	
	if bkfile:
		dataHost.CATbackgroundData(bkfilename)
	
	#dataHost.setAllCAT(0.002)
	if CATbaseline:
		dataHost.setCATbaseline(0.002)
	if MOTbaseline:
		dataHost.setBaseline(0.002)
	if loadFit:
		dataHost.setLoading(0.002)
	if initRFit and loadFit:
		dataHost.initFit, dataHost.initX = dataHost.setInitialLoad(0.002)
	if deloadFit:
		dataHost.setDeloading(0.002)
		dataHost.plotDeloadFit(run_path)# TODO: currently just stores the deloading times and voltages
	if reloadFit:
		dataHost.setReloadVolt(0.002)

	if CATbaseline and MOTbaseline and loadFit and reloadFit:
	# steady state ratio fraction
		dataHost.ratio = dataHost.reloadVolt / dataHost.motSS

		dataHost.ratioErr = dataHost.ratio * ((dataHost.reloadVoltErr/dataHost.reloadVolt)**2 + (dataHost.motSSErr/dataHost.motSS)**2)**(0.5)
	
		# if abs(dataHost.ratioErr / dataHost.ratio) > 0.1:
		#     dataHost.ratioErr = abs(0.015*dataHost.ratio)
		if dataHost.ratioErr < 0.001:
			dataHost.ratioErr = 0.001
		if dataHost.ratio < 0:
			dataHost.ratio = 0
			
		# TODO: this information is useless
	print('File loaded: RFmin = {} MHz, t_mt = {:.3f} s.'.format(dataHost.settings['fmin'], dataHost.settings['wait_mtrap']))
	
	resultDict = dataHost.getResults(run_path, store=storeFitResults)
 
	if plot:
		dataHost.storeFits(run_path, combined=True, separate=True)
  
	return resultDict, dataHost.settings
		
def get_timestamp(run_path):
	timestamp = datetime.strptime(os.path.split(run_path)[-1].split('_')[0], dateformat)
	return timestamp

def extract_fit(run_path, plot=True, cache_failed=True, cache_all=True):
	"""Gather relevant data from each measurement run

	Args:
		run_path : absolute path to the run directory
		plot (bool, optional): plot fits. Defaults to True.
		cache_failed (bool, optional): Cache failed fits. If false, refit. Doesn't refit non-failed fits. Defaults to True.
		cache_all (bool, optional): If false, ignore any cached fit_results. Defaults to True.

	Returns:
		a 3-tuple (fit_results, settings, timestamp)
	"""
	fit_results, settings, timestamp = {}, {}, None
	
	if not os.path.isdir(run_path):
		return fit_results, settings, timestamp  # directory is not a run directory

	try:
		timestamp = get_timestamp(run_path)
  
  	# TODO: specify which error to catch
	except Exception as e:
		print("Error extracting timestamp from: ", run_path)
		print(traceback.format_exc())
	
	MAT_fit_cache_path = os.path.join(run_path, 'resultDict.txt')
 
	if not os.path.exists(MAT_fit_cache_path) or not cache_all:
		
		try: 
			fit_results, settings = save_fit_results(run_path, plot=plot)
		except Exception as e:
			print(traceback.format_exc())
			print("Fitting ERROR at ", os.path.basename(run_path), '\n')

			with open(MAT_fit_cache_path, 'w') as f:
				f.write(str('MAT fit failed'))
		
	else:
		print("Accessing cached results from :", os.path.basename(run_path))
  
		fit_results = open(MAT_fit_cache_path, 'r').read()

		if fit_results == 'MAT fit failed':
			if not cache_failed:
				# fit regardless of cached result
				try: 
					fit_results, settings = save_fit_results(run_path, plot=plot)
				except Exception as e:
					print(traceback.format_exc())
					print("Fitting ERROR at ", os.path.basename(run_path), '\n')
	
					with open(MAT_fit_cache_path, 'w') as f:
						f.write(str('MAT fit failed'))	
			else:
				print("Failed fit at :", os.path.basename(run_path))
				fit_results = {}

		else:
			fit_results = eval(open(MAT_fit_cache_path, 'r').read())
   
			settingsname = os.path.join(run_path, 'Settings.txt')
			settings = eval(open(settingsname, 'r').read())
   
	return fit_results, settings, timestamp


def get_row(run_path, **kwargs):
			
   fit_results, settings, timestamp = extract_fit(run_path, **kwargs)
   row = {**fit_results, **settings, **{'timestamp':timestamp}}
   
   return row

   
def get_data_frame(data_dir, parallel=True, in_process_run=False, **kwargs):
   
	run_path_arr = []
	rows = []
	
	for relative_path in os.listdir(data_dir):
		run_path_arr.append(os.path.join(data_dir, relative_path))
  
	if in_process_run:
		run_path_arr.pop()

	run_path_arr = sorted(run_path_arr)
	
	if parallel:
		with Pool(4) as p:
			rows = list(tqdm(p.imap(partial(get_row, **kwargs), run_path_arr), total=len(run_path_arr)))
	
	else:
		for run_path in tqdm(run_path_arr):
		   rows.append(get_row(run_path, **kwargs))
	return pd.DataFrame.from_dict(rows)

def add_wavemeter_data(df, wmeter_csv_path, window_size=100, num_rows=50):


	"""Extract unique frequnecy values from wavemeter data
	Returns:
		unique_levels (list): unique frequency values in wavemeter data
	"""
 
	# TODO: modify dataframe in place with frequency data
 
	wdata = pd.read_csv(wmeter_csv_path, skiprows=2)
	wdata.dropna(inplace=True)
	freq_data = np.array(wdata.iloc[:, 0])
	try:
		freq_data = np.array([float(item) for item in freq_data if item.replace('.','').isdigit()])
	except Exception as e:
		print(e)
 
	max_freq = freq_data.max()
	min_freq = freq_data.min()

	return freq_data, max_freq, min_freq

def plot_results(ax, dfs, max_freq, min_freq=0.0, mfc='red', fmt='o', ms=5, save_folder=False, xscale=1.0, yscale=1.0, **kwargs):
	FREQVSVOLT = 221.0 
	FREQVSCURR = 1.13
	
	if not type(dfs) == list:
		freqs = ((max_freq-PUMP_FREQUENCY)-(dfs.dropna()['tempV']-dfs.dropna()['tempV'].min())*FREQVSVOLT- (dfs.dropna()['currV']-dfs.dropna()['currV'].min())*FREQVSCURR)*xscale
		dfs=[dfs]
  
	plt.gcf().set_dpi(300)
	
	for df in dfs:
		df = df.dropna()
		ax.errorbar(freqs, 
			   df['ratio']*yscale,
			   yerr=df['ratioErr'],
			   fmt=fmt, mfc=mfc, color='black', ms=ms, **kwargs)
		ax.set_ylabel(r'$\mathbf{\frac{V_{ss, cat}}{V_{ss}}} $ ', **labeldict)
		ax.set_xlabel(r'$\Delta $ (GHz)', **labeldict)
	
	if save_folder:
		plt.savefig(os.path.join(save_folder, 'ratio_vs_freq.png'))
	
	return freqs, ax
  
	#return plt.gca(), plt.gcf()
	#plt.show()

def plot_spline_fit(ax, x, y, s=1, yerr=None, color='black', scolor='black',figsize=(12,5), save_folder=None, title='',alpha=0.5,dpi=200, label='plot', fig=None,**kwargs):
	from scipy.interpolate import splev, splrep
	xnew = np.linspace(min(x), max(x), 3*len(x) )

	y = [b for a,b in sorted(zip(x,y), key=lambda pair: pair[0])]
	if yerr is not None:
		yerr = [b for a,b in sorted(zip(x,yerr), key=lambda pair: pair[0])]
 
	x = sorted(x)


	spl = splrep(x, y, s=s)
	ynew = splev(xnew, spl)

	if fig is None:
		plt.gcf().set_dpi(dpi)
		plt.gcf().set_size_inches(figsize)
	else:
		fig.set_dpi(dpi)
		fig.set_size_inches(figsize)
	if yerr is not None:
		ax.errorbar(x, y, yerr=yerr, fmt='o', **kwargs)
	else:
		ax.plot(x,y, 'o', **kwargs)
	ax.plot(xnew, ynew, '-', color=scolor, alpha=alpha, label=label, **kwargs)
 
	ax.set_ylabel(r'$\mathbf{\frac{V_{ss, cat}}{V_{ss}}} $ ', **labeldict)
	ax.set_xlabel(r'$\Delta $ (GHz)', **labeldict)
 
	ax.set_title(title, **titledict)
	if save_folder:
		plt.savefig(os.path.join(save_folder, 'spline_ratio_vs_freq.png'))
	
	return ax
 
def plot_polyfit(x_data, y_data, spline_degree):
	
	coefficients = np.polyfit(x_data, y_data, spline_degree)

	x_interp = np.linspace(min(x_data), max(x_data), 100)
	y_interp = np.polyval(coefficients, x_interp)

	plt.scatter(x_data, y_data, label='Original Data')
	plt.plot(x_interp, y_interp, label='Polynomial Interpolation (Degree={})'.format(spline_degree))

def collect_plots(source, destination, plot_name):
	print(f'Collecting plots from {os.path.basename(source)}')
	import shutil

	os.makedirs(destination, exist_ok=True)

	plot_files = []
	for root, dirs, files in os.walk(source):
		
		for file in files:
			if file == plot_name:
				plot_files.append(os.path.join(root, file))

	for i, plot_file in enumerate(plot_files, start=0):
		new_filename = f'{i}{plot_name}'
		destination_path = os.path.join(destination, new_filename)
		shutil.copy(plot_file, destination_path)

def create_GIF(images_folder, image_name):
	import imageio
	with imageio.get_writer(os.path.join(images_folder, f'{image_name}movie.gif'), mode='I', duration=0.5) as writer:
		for filename in os.listdir(images_folder):
			if image_name in filename:
				image = imageio.imread(os.path.join(images_folder, filename))
				writer.append_data(image)

def staircase_fit(data, peak_height=0.1, distance=100, data_offset=1, window_size=1, inc_final_peak=True):
	def moving_average(arr, window_size):
		weights = np.ones(window_size) / window_size
		return np.convolve(arr, weights, mode='valid')

	convdata1 = moving_average(data, window_size)
	convdata2 = moving_average(data[data_offset+1:], window_size )
	final = convdata1[:len(convdata2)]-convdata2
	# plt.plot(data)
	# plt.plot(convdata1)
	# plt.plot(convdata2)
	# plt.show()
	# plt.plot(final)

	
	from scipy.signal import find_peaks
	x= final
	peaks, _ = find_peaks(x, height=peak_height, distance=distance)
	peaks = np.insert(peaks, 0, 0)
	levels = []
	plot_arr = []
	for i, peak in enumerate(peaks):
		
		if i < len(peaks) - 1:
			temp = data[ peaks[i]:peaks[i+1] ]
			plot_arr.extend( np.ones_like(temp)*np.mean(temp))
			levels.append(np.mean(temp))
	if inc_final_peak:
		temp = data[peaks[-1]:]
		levels.append(np.mean(temp))
		plot_arr.extend( np.ones_like(temp)*np.mean(temp))
  
	plt.plot(x)
	plt.plot(peaks, x[peaks], "x")
	plt.plot(np.zeros_like(x), "--", color="gray")
	plt.show()
	
	plt.plot(data)
	plt.plot(np.ravel((plot_arr)))
	plt.show()
	plt.close()
	plt.plot(np.array(levels)[np.where(abs(np.diff(levels))>0.05)[0]], 'o', ms=5)
 
	return levels

def load_single_run(run_path):
	filename = os.path.join(run_path, 'data.csv')
	bkfilename = os.path.join(run_path, 'data_oldPD.csv')
	settingsname = os.path.join(run_path, 'Settings.txt')
	
	dh1 = MTdataHost(SAMPLE_RATE)
	dh1.loadCATdata(fileName=filename, settingsName=settingsname)
	
	return dh1
	
def load_mega_run(MEASURE_FOLDER, groupbyKey, titleKey, plot=True, save_plots=False, max_freq=384182.5, **kwargs):
    # TODO: use the plot flag to do something?
    
	df = get_data_frame(MEASURE_FOLDER,
						**kwargs)
	dfc= df.copy()
	#df.dropna(inplace=True)

	df_grouped = df.groupby(by=groupbyKey)
	min_ratios = df_grouped['ratio'].min()

	groups = dict(list(df_grouped))
	dfs = [df for df in groups.values()]

	# plotting ratio vs freq
	max_freqs = [max_freq]*len(dfs)
	zipped_data = list(zip(dfs, max_freqs))
	fig1, ax = plt.subplots()
	for i, (df, max_freq)  in enumerate(zipped_data[:]):
		data = df
		freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.0, ms=5, figsize=(10, 10), label=f"{groupbyKey} = { data.iloc[10][groupbyKey] :.2f}", linewidth=2.5)


	plt.legend()
	
	plt.title(f'Loss Features, {titleKey} = {data[titleKey].mean():.2f}', **titledict)
	plt.show()

	#---------------------------------------------------
	fig2 = plt.figure(2)
	x = [df[groupbyKey].mean() for df in dfs]
	y = [(df['ratio'].max() - df['ratio'].min()) for df in dfs]
	plt.plot( x, y ,'-o')
	plt.xlabel(groupbyKey)
	plt.ylabel(r'SNR $ = V_{ss, off} - V_{ss, on}$ ')
	plt.title(f'SNR Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict)
	plt.show()
	
	#---------------------------------------------------------
	fig3=plt.figure(3)
	for i, df  in enumerate(dfs[:]):
		data = df
		freqs = ((384182.5-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
		freqs = sorted(freqs)
		plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{groupbyKey}={data.iloc[10][groupbyKey]:.2f}")
	plt.legend()
	plt.xlabel(r'$\Delta$ (GHz)')
	plt.ylabel(r'$\beta_{\mathrm{eff}}$ ')
	plt.title(f'2-body Decay Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict) 
	plt.show()

	if save_plots:
		fig1.savefig(os.path.join(MEASURE_FOLDER, 'lossFeatures.png'))
		fig2.savefig(join(MEASURE_FOLDER, 'SNRplot.png'), dpi=200)
		fig3.savefig(join(MEASURE_FOLDER, 'betaVsFreq.png'), dpi=200) 
	return dfc


if __name__ == '__main__':
	# run_path = r"C:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\CATMeasurements\testPArun9\16-53-10"
	# filename = os.path.join(run_path, 'data.csv')
	# bkfilename = os.path.join(run_path, 'data_oldPD.csv')
	# settingsname = os.path.join(run_path, 'Settings.txt')
	
	# dh1 = MTdataHost(SAMPLE_RATE)
	# dh1.loadCATdata(fileName=filename, settingsName=settingsname)
	
	# dh1.setAllCAT(0.002)
 
 	#dh1.CATbackgroundData(bkfilename)
	
	pass


folders = [join(EXP_FOLDER, 'RepumpSizeTrapDepthStudy', path ) for path in ['PAsmallRepumpSizeTest', 'PAsmallRepumpSizeTest2', 'PAsmallRepumpSizeTest3', 'PAwideRepumpSizeTest', 'PAwideRepumpSizeTest2']]

dfs = [get_data_frame(measure_folder, cache_all=False, plot=False, parallel=False) for measure_folder in [folders[1]]]
labels = ['Narrow Beam' , 'Narrow Beam', 'Narrow Beam', 'Wide Beam', 'Wide Beam' ][1:]
max_freqs = [384182.6]*len(dfs)
zipped_data = list(zip(dfs, max_freqs))
for i, (df, max_freq)  in enumerate(zipped_data[:]):
    data=df
    freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
    betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
    freqs = sorted(freqs)
    plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{labels[i]}")
    plt.xlabel(r'$\Delta$ (GHz)')
    plt.ylabel(r'$\beta$')
    
plt.legend()

  4%|▍         | 1/23 [00:01<00:35,  1.63s/it]

File loaded: RFmin = 49 MHz, t_mt = 1.050 s.

  9%|▊         | 2/23 [00:02<00:28,  1.37s/it]

File loaded: RFmin = 49 MHz, t_mt = 1.050 s.

 13%|█▎        | 3/23 [00:03<00:24,  1.22s/it]

File loaded: RFmin = 49 MHz, t_mt = 1.050 s.

 17%|█▋        | 4/23 [00:05<00:24,  1.27s/it]

File loaded: RFmin = 49 MHz, t_mt = 1.050 s.

 22%|██▏       | 5/23 [00:06<00:26,  1.45s/it]

File loaded: RFmin = 49 MHz, t_mt = 1.050 s.


folders = [join(EXP_FOLDER, 'RepumpSizeTrapDepthStudy', path ) for path in ['PAsmallRepumpSizeTest', 'PAsmallRepumpSizeTest2', 'PAsmallRepumpSizeTest3', 'PAwideRepumpSizeTest', 'PAwideRepumpSizeTest2']]

dfs = [get_data_frame(measure_folder, cache_all=False, plot=False, parallel=False) for measure_folder in folders[1]]
labels = ['Narrow Beam' , 'Narrow Beam', 'Narrow Beam', 'Wide Beam', 'Wide Beam' ][1:]
max_freqs = [384182.6]*len(dfs)
zipped_data = list(zip(dfs, max_freqs))
for i, (df, max_freq)  in enumerate(zipped_data[:]):
    data=df
    freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
    betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
    freqs = sorted(freqs)
    plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{labels[i]}")
    plt.xlabel(r'$\Delta$ (GHz)')
    plt.ylabel(r'$\beta$')
    
plt.legend()

---------------------------------------------------------------------------
FileNotFoundError                         Traceback (most recent call last)
Cell In[3], line 3
      1 folders = [join(EXP_FOLDER, 'RepumpSizeTrapDepthStudy', path ) for path in ['PAsmallRepumpSizeTest', 'PAsmallRepumpSizeTest2', 'PAsmallRepumpSizeTest3', 'PAwideRepumpSizeTest', 'PAwideRepumpSizeTest2']]
----> 3 dfs = [get_data_frame(measure_folder, cache_all=False, plot=False, parallel=False) for measure_folder in folders[1]]
      4 labels = ['Narrow Beam' , 'Narrow Beam', 'Narrow Beam', 'Wide Beam', 'Wide Beam' ][1:]
      5 max_freqs = [384182.6]*len(dfs)

Cell In[3], line 3, in <listcomp>(.0)
      1 folders = [join(EXP_FOLDER, 'RepumpSizeTrapDepthStudy', path ) for path in ['PAsmallRepumpSizeTest', 'PAsmallRepumpSizeTest2', 'PAsmallRepumpSizeTest3', 'PAwideRepumpSizeTest', 'PAwideRepumpSizeTest2']]
----> 3 dfs = [get_data_frame(measure_folder, cache_all=False, plot=False, parallel=False) for measure_folder in folders[1]]
      4 labels = ['Narrow Beam' , 'Narrow Beam', 'Narrow Beam', 'Wide Beam', 'Wide Beam' ][1:]
      5 max_freqs = [384182.6]*len(dfs)

c:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\analysis\analysis_test.py in line 423, in get_data_frame(data_dir, parallel, in_process_run, **kwargs)
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=419'>420</a> run_path_arr = []
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=420'>421</a> rows = []
--> <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=422'>423</a> for relative_path in os.listdir(data_dir):
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=423'>424</a> 	run_path_arr.append(os.path.join(data_dir, relative_path))
    <a href='file:///c%3A/Users/svars/OneDrive/Desktop/UBC%20Lab/CATExperiment/analysis/analysis_test.py?line=425'>426</a> if in_process_run:

FileNotFoundError: [WinError 3] The system cannot find the path specified: 'C'


folders = [join(EXP_FOLDER, 'RepumpSizeTrapDepthStudy', path ) for path in ['PAsmallRepumpSizeTest', 'PAsmallRepumpSizeTest2', 'PAsmallRepumpSizeTest3', 'PAwideRepumpSizeTest', 'PAwideRepumpSizeTest2']]

dfs = [get_data_frame(measure_folder, cache_all=False, plot=False, parallel=False) for measure_folder in folders][1:]
labels = ['Narrow Beam' , 'Narrow Beam', 'Narrow Beam', 'Wide Beam', 'Wide Beam' ][1:]
max_freqs = [384182.6]*len(dfs)
zipped_data = list(zip(dfs, max_freqs))
for i, (df, max_freq)  in enumerate(zipped_data[:]):
    data=df
    freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
    betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
    freqs = sorted(freqs)
    plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{labels[i]}")
    plt.xlabel(r'$\Delta$ (GHz)')
    plt.ylabel(r'$\beta$')
    
plt.legend()

  4%|▍         | 1/24 [00:01<00:42,  1.87s/it]

File loaded: RFmin = 49 MHz, t_mt = 1.050 s.

  8%|▊         | 2/24 [00:03<00:37,  1.72s/it]

File loaded: RFmin = 49 MHz, t_mt = 1.050 s.

 12%|█▎        | 3/24 [00:04<00:30,  1.46s/it]

File loaded: RFmin = 49 MHz, t_mt = 1.050 s.

 17%|█▋        | 4/24 [00:06<00:30,  1.55s/it]

File loaded: RFmin = 49 MHz, t_mt = 1.050 s.

 21%|██        | 5/24 [00:07<00:28,  1.48s/it]

File loaded: RFmin = 49 MHz, t_mt = 1.050 s.


folders = [join(EXP_FOLDER, 'RepumpSizeTrapDepthStudy', path ) for path in ['PAsmallRepumpSizeTest', 'PAsmallRepumpSizeTest2', 'PAsmallRepumpSizeTest3', 'PAwideRepumpSizeTest', 'PAwideRepumpSizeTest2']]

dfs = [get_data_frame(measure_folder, cache_all=True, plot=False, parallel=True) for measure_folder in folders]
labels = ['Narrow Beam' , 'Narrow Beam', 'Narrow Beam', 'Wide Beam', 'Wide Beam' ][1:]
max_freqs = [384182.6]*len(dfs)
zipped_data = list(zip(dfs, max_freqs))
for i, (df, max_freq)  in enumerate(zipped_data[:]):
    data=df
    freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
    betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
    freqs = sorted(freqs)
    plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{labels[i]}")
    plt.xlabel(r'$\Delta$ (GHz)')
    plt.ylabel(r'$\beta$')
    
plt.legend()

100%|██████████| 24/24 [00:04<00:00,  5.65it/s]
100%|██████████| 23/23 [00:02<00:00,  8.29it/s]
100%|██████████| 23/23 [00:02<00:00,  7.81it/s]
100%|██████████| 24/24 [00:02<00:00,  8.60it/s]
100%|██████████| 22/22 [00:02<00:00,  7.60it/s]

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
Cell In[5], line 12
     10 betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
     11 freqs = sorted(freqs)
---> 12 plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{labels[i]}")
     13 plt.xlabel(r'$\Delta$ (GHz)')
     14 plt.ylabel(r'$\beta$')

IndexError: list index out of range


dfs[0]['motSS'].mean()

0.019941870561213097


dfs[1]['motSS'].mean()

0.027642133084095893


dfs[2]['motSS'].mean()

0.02760240936147823


dfs[3]['motSS'].mean()

0.029971958996811494


dfs[4]['motSS'].mean()

0.02933410609208271


dfs[4]['initMOTR'].mean()

0.001727234594410109


dfs[3]['initMOTR'].mean()

0.0016056072777040215


dfs[2]['initMOTR'].mean()

0.0015359611649215237


dfs[1]['initMOTR'].mean()

0.001633255664005241


dfs[0]['initMOTR'].mean()

0.001137303729569656


dfs[0]['backgroundVolt'].mean()

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\core\indexes\base.py:3802, in Index.get_loc(self, key, method, tolerance)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3800'>3801</a> try:
-> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3801'>3802</a>     return self._engine.get_loc(casted_key)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3802'>3803</a> except KeyError as err:

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\_libs\index.pyx:138, in pandas._libs.index.IndexEngine.get_loc()

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\_libs\index.pyx:165, in pandas._libs.index.IndexEngine.get_loc()

File pandas\_libs\hashtable_class_helper.pxi:5745, in pandas._libs.hashtable.PyObjectHashTable.get_item()

File pandas\_libs\hashtable_class_helper.pxi:5753, in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: 'backgroundVolt'

The above exception was the direct cause of the following exception:

KeyError                                  Traceback (most recent call last)
Cell In[16], line 1
----> 1 dfs[0]['backgroundVolt'].mean()

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\core\frame.py:3807, in DataFrame.__getitem__(self, key)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3804'>3805</a> if self.columns.nlevels > 1:
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3805'>3806</a>     return self._getitem_multilevel(key)
-> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3806'>3807</a> indexer = self.columns.get_loc(key)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3807'>3808</a> if is_integer(indexer):
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3808'>3809</a>     indexer = [indexer]

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\core\indexes\base.py:3804, in Index.get_loc(self, key, method, tolerance)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3801'>3802</a>     return self._engine.get_loc(casted_key)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3802'>3803</a> except KeyError as err:
-> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3803'>3804</a>     raise KeyError(key) from err
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3804'>3805</a> except TypeError:
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3805'>3806</a>     # If we have a listlike key, _check_indexing_error will raise
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3806'>3807</a>     #  InvalidIndexError. Otherwise we fall through and re-raise
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3807'>3808</a>     #  the TypeError.
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3808'>3809</a>     self._check_indexing_error(key)

KeyError: 'backgroundVolt'


dfs[0]['baselineVolt'].mean()

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\core\indexes\base.py:3802, in Index.get_loc(self, key, method, tolerance)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3800'>3801</a> try:
-> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3801'>3802</a>     return self._engine.get_loc(casted_key)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3802'>3803</a> except KeyError as err:

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\_libs\index.pyx:138, in pandas._libs.index.IndexEngine.get_loc()

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\_libs\index.pyx:165, in pandas._libs.index.IndexEngine.get_loc()

File pandas\_libs\hashtable_class_helper.pxi:5745, in pandas._libs.hashtable.PyObjectHashTable.get_item()

File pandas\_libs\hashtable_class_helper.pxi:5753, in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: 'baselineVolt'

The above exception was the direct cause of the following exception:

KeyError                                  Traceback (most recent call last)
Cell In[17], line 1
----> 1 dfs[0]['baselineVolt'].mean()

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\core\frame.py:3807, in DataFrame.__getitem__(self, key)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3804'>3805</a> if self.columns.nlevels > 1:
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3805'>3806</a>     return self._getitem_multilevel(key)
-> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3806'>3807</a> indexer = self.columns.get_loc(key)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3807'>3808</a> if is_integer(indexer):
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3808'>3809</a>     indexer = [indexer]

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\core\indexes\base.py:3804, in Index.get_loc(self, key, method, tolerance)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3801'>3802</a>     return self._engine.get_loc(casted_key)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3802'>3803</a> except KeyError as err:
-> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3803'>3804</a>     raise KeyError(key) from err
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3804'>3805</a> except TypeError:
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3805'>3806</a>     # If we have a listlike key, _check_indexing_error will raise
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3806'>3807</a>     #  InvalidIndexError. Otherwise we fall through and re-raise
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3807'>3808</a>     #  the TypeError.
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3808'>3809</a>     self._check_indexing_error(key)

KeyError: 'baselineVolt'


folders = [join(EXP_FOLDER, 'expandedBeam', path ) for path in ['testNewBeam1']]

dfs = [get_data_frame(measure_folder, cache_all=False, plot=False, parallel=False) for measure_folder in folders]
max_freqs = [384182.6]*len(dfs)
zipped_data = list(zip(dfs, max_freqs))
for i, (df, max_freq)  in enumerate(zipped_data):
    data=df
    freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
    betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
    freqs = sorted(freqs)
    plt.plot(freqs, betaPAs, 'o-', ms=5)
    plt.xlabel(r'$\Delta$ (GHz)')
    plt.ylabel(r'$\beta$')
    
plt.legend()

  5%|▍         | 1/21 [00:01<00:27,  1.36s/it]

File loaded: RFmin = 49 MHz, t_mt = 1.050 s.

 10%|▉         | 2/21 [00:02<00:24,  1.28s/it]

File loaded: RFmin = 49 MHz, t_mt = 1.050 s.

 14%|█▍        | 3/21 [00:04<00:24,  1.39s/it]

File loaded: RFmin = 49 MHz, t_mt = 1.050 s.

 19%|█▉        | 4/21 [00:05<00:25,  1.50s/it]

File loaded: RFmin = 49 MHz, t_mt = 1.050 s.

 24%|██▍       | 5/21 [00:07<00:23,  1.49s/it]

File loaded: RFmin = 49 MHz, t_mt = 1.050 s.


MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'expandedBeam', 'testNewBeam1')
df = get_data_frame(MEASURE_FOLDER)
data = df.dropna()

max_freq = 384182.5

freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-df['tempV'].min())*FREQVSVOLT- (data['currV']-df['currV'].min())*FREQVSCURR)

fig, ax = plt.subplots()
plot_spline_fit(ax=ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'], 
                s=0.0, save_folder=MEASURE_FOLDER, 
                mfc='red', color='black', scolor='black',  
                title='')
plt.show()
plt.close()

betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
freqs = sorted(freqs)
plt.plot(freqs, betaPAs, 'o-', ms=5)
plt.legend()
plt.xlabel(r'$\Delta$ (GHz)')
plt.ylabel(r'$\beta_{\mathrm{eff}}$ ')
#plt.savefig(join(MEASURE_FOLDER, 'betaVsFreq.png'), dpi=200) 
plt.title(f"2-body Decay Plot {' '}", **titledict) 
plt.show()
plt.close()

100%|██████████| 22/22 [00:03<00:00,  6.12it/s]

No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.


MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'expandedBeam', 'testNewBeam1')
df = get_data_frame(MEASURE_FOLDER)
data = df

max_freq = 384182.5

freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-df['tempV'].min())*FREQVSVOLT- (data['currV']-df['currV'].min())*FREQVSCURR)

fig, ax = plt.subplots()
plot_spline_fit(ax=ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'], 
                s=0.0, save_folder=MEASURE_FOLDER, 
                mfc='red', color='black', scolor='black',  
                title='')
plt.show()
plt.close()

betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
freqs = sorted(freqs)
plt.plot(freqs, betaPAs, 'o-', ms=5)
plt.legend()
plt.xlabel(r'$\Delta$ (GHz)')
plt.ylabel(r'$\beta_{\mathrm{eff}}$ ')
#plt.savefig(join(MEASURE_FOLDER, 'betaVsFreq.png'), dpi=200) 
plt.title(f"2-body Decay Plot {' '}", **titledict) 
plt.show()
plt.close()

100%|██████████| 22/22 [00:03<00:00,  6.68it/s]

---------------------------------------------------------------------------
error                                     Traceback (most recent call last)
Cell In[29], line 10
      7 freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-df['tempV'].min())*FREQVSVOLT- (data['currV']-df['currV'].min())*FREQVSCURR)
      9 fig, ax = plt.subplots()
---> 10 plot_spline_fit(ax=ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'], 
     11                 s=0.0, save_folder=MEASURE_FOLDER, 
     12                 mfc='red', color='black', scolor='black',  
     13                 title='')
     14 plt.show()
     15 plt.close()

Cell In[22], line 506, in plot_spline_fit(ax, x, y, s, yerr, color, scolor, figsize, save_folder, title, alpha, dpi, label, fig, **kwargs)
    501 	yerr = [b for a,b in sorted(zip(x,yerr), key=lambda pair: pair[0])]
    503 x = sorted(x)
--> 506 spl = splrep(x, y, s=s)
    507 ynew = splev(xnew, spl)
    509 if fig is None:

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\scipy\interpolate\_fitpack_py.py:288, in splrep(x, y, w, xb, xe, k, task, s, t, full_output, per, quiet)
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/interpolate/_fitpack_py.py?line=157'>158</a> def splrep(x, y, w=None, xb=None, xe=None, k=3, task=0, s=None, t=None,
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/interpolate/_fitpack_py.py?line=158'>159</a>            full_output=0, per=0, quiet=1):
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/interpolate/_fitpack_py.py?line=159'>160</a>     """
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/interpolate/_fitpack_py.py?line=160'>161</a>     Find the B-spline representation of a 1-D curve.
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/interpolate/_fitpack_py.py?line=161'>162</a> 
   (...)
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/interpolate/_fitpack_py.py?line=285'>286</a> 
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/interpolate/_fitpack_py.py?line=286'>287</a>     """
--> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/interpolate/_fitpack_py.py?line=287'>288</a>     res = _impl.splrep(x, y, w, xb, xe, k, task, s, t, full_output, per, quiet)
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/interpolate/_fitpack_py.py?line=288'>289</a>     return res

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\scipy\interpolate\_fitpack_impl.py:494, in splrep(x, y, w, xb, xe, k, task, s, t, full_output, per, quiet)
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/interpolate/_fitpack_impl.py?line=490'>491</a>     raise TypeError("must call with task=1 only after"
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/interpolate/_fitpack_impl.py?line=491'>492</a>                     " call with task=0,-1") from e
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/interpolate/_fitpack_impl.py?line=492'>493</a> if not per:
--> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/interpolate/_fitpack_impl.py?line=493'>494</a>     n, c, fp, ier = dfitpack.curfit(task, x, y, w, t, wrk, iwrk,
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/interpolate/_fitpack_impl.py?line=494'>495</a>                                     xb, xe, k, s)
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/interpolate/_fitpack_impl.py?line=495'>496</a> else:
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/scipy/interpolate/_fitpack_impl.py?line=496'>497</a>     n, c, fp, ier = dfitpack.percur(task, x, y, w, t, wrk, iwrk, k, s)

error: (xe>=x[m-1]) failed for 2nd keyword xe: curfit:xe=nan

df


df[:-2]


columns_with_nan = df.columns[df.isna().any()].tolist()

print("Columns with NaN values:", columns_with_nan)

Columns with NaN values: ['sampleRate', 'extraTime', 'timeHold', 'timeBaseline', 'timeTest', 'timeLoad', 'timeF1', 'offset', 'baseVolt', 'BaseVoltErr', 'motSS', 'motSSErr', 'MTvolt', 'MTvoltErr', 'bkBool', 'CATbkBool', 'timeDeload', 'timeReload', 'noLightBackground', 'tCATbackground', 'tLoad', 'tDeload', 'deloadFitTime', 'tReload', 'tBaseline', 'CATbackground1', 'CATbackground2', 'CATbackgroundVolt', 'CATbackgroundVoltErr', 'base1', 'base2', 'std', 'motA', 'motR', 'motRErr', 'motFitR', 'motFitRErr', 'initStartInd', 'initEndInd', 'initMOTR', 'initMOTRErr', 'deloadStartInd', 'deloadEndInd', 'betaPA', 'betaPAErr', 'tDeloadEnd', 'reloadStartInd', 'reloadEndInd', 'reloadVolt', 'reloadVoltErr', 'reloadVoltpt', 'reloadVoltT', 'linear_reload_fit', 'ratio', 'ratioErr', 'detuning_F1', 'detuning_F2', 'wait_cool', 'bool_cooling', 'state', 'wait_image', 'wait_Load', 'wait_hfine_pump_F1', 'wait_hfine_pump_F2', 'wait_baseline', 'dtpump', 'dtrepump', 'coil_set', 'OP_coil', 'wait_mtrap', 'pump_MT_ampl', 'repump_MT_ampl', 'MOT_coil_set', 'pump_AOM_freq', 'repump_AOM_freq', 'OP_AOM_freq', 'pump_ampl', 'repump_ampl', 'OP_ampl', 'pump_reference', 'repump_reference', 'RFtime', 'fmin', 'fmax', 'RFampl', 'RFsweep', 'wait_pump', 'wait_repump', 'load_linfit_1', 'load_linfit_2', 'wait_background', 'test_coil_set', 'test_MOT_coil_set', 'test_pump_AOM_freq', 'test_repump_AOM_freq', 'test_pump_ampl', 'test_repump_ampl', 'test_pump_reference', 'test_repump_reference', 'test_MOT_coil', 'numPowers', 'comp_time', 'repump_time', 'exptime', 'clearbool', 'wallbool', 't_ramp', 'clearfrac', 'precut_fmin', 'precut_t', 'filtertime', 'master_clear', 'tempV', 'currV', 'cat_AOM_freq', 'cat_AOM_ampl', 'cat_deload_t', 'MOT_reload_t', 'timestamp']


columns_with_nan = df[-2].columns[df.isna().any()].tolist()

print("Columns with NaN values:", columns_with_nan)

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\core\indexes\base.py:3802, in Index.get_loc(self, key, method, tolerance)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3800'>3801</a> try:
-> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3801'>3802</a>     return self._engine.get_loc(casted_key)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3802'>3803</a> except KeyError as err:

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\_libs\index.pyx:138, in pandas._libs.index.IndexEngine.get_loc()

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\_libs\index.pyx:165, in pandas._libs.index.IndexEngine.get_loc()

File pandas\_libs\hashtable_class_helper.pxi:5745, in pandas._libs.hashtable.PyObjectHashTable.get_item()

File pandas\_libs\hashtable_class_helper.pxi:5753, in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: -2

The above exception was the direct cause of the following exception:

KeyError                                  Traceback (most recent call last)
Cell In[33], line 1
----> 1 columns_with_nan = df[-2].columns[df.isna().any()].tolist()
      3 print("Columns with NaN values:", columns_with_nan)

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\core\frame.py:3807, in DataFrame.__getitem__(self, key)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3804'>3805</a> if self.columns.nlevels > 1:
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3805'>3806</a>     return self._getitem_multilevel(key)
-> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3806'>3807</a> indexer = self.columns.get_loc(key)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3807'>3808</a> if is_integer(indexer):
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/frame.py?line=3808'>3809</a>     indexer = [indexer]

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\pandas\core\indexes\base.py:3804, in Index.get_loc(self, key, method, tolerance)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3801'>3802</a>     return self._engine.get_loc(casted_key)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3802'>3803</a> except KeyError as err:
-> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3803'>3804</a>     raise KeyError(key) from err
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3804'>3805</a> except TypeError:
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3805'>3806</a>     # If we have a listlike key, _check_indexing_error will raise
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3806'>3807</a>     #  InvalidIndexError. Otherwise we fall through and re-raise
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3807'>3808</a>     #  the TypeError.
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/pandas/core/indexes/base.py?line=3808'>3809</a>     self._check_indexing_error(key)

KeyError: -2


columns_with_nan = df[:-2].columns[df.isna().any()].tolist()

print("Columns with NaN values:", columns_with_nan)

Columns with NaN values: ['sampleRate', 'extraTime', 'timeHold', 'timeBaseline', 'timeTest', 'timeLoad', 'timeF1', 'offset', 'baseVolt', 'BaseVoltErr', 'motSS', 'motSSErr', 'MTvolt', 'MTvoltErr', 'bkBool', 'CATbkBool', 'timeDeload', 'timeReload', 'noLightBackground', 'tCATbackground', 'tLoad', 'tDeload', 'deloadFitTime', 'tReload', 'tBaseline', 'CATbackground1', 'CATbackground2', 'CATbackgroundVolt', 'CATbackgroundVoltErr', 'base1', 'base2', 'std', 'motA', 'motR', 'motRErr', 'motFitR', 'motFitRErr', 'initStartInd', 'initEndInd', 'initMOTR', 'initMOTRErr', 'deloadStartInd', 'deloadEndInd', 'betaPA', 'betaPAErr', 'tDeloadEnd', 'reloadStartInd', 'reloadEndInd', 'reloadVolt', 'reloadVoltErr', 'reloadVoltpt', 'reloadVoltT', 'linear_reload_fit', 'ratio', 'ratioErr', 'detuning_F1', 'detuning_F2', 'wait_cool', 'bool_cooling', 'state', 'wait_image', 'wait_Load', 'wait_hfine_pump_F1', 'wait_hfine_pump_F2', 'wait_baseline', 'dtpump', 'dtrepump', 'coil_set', 'OP_coil', 'wait_mtrap', 'pump_MT_ampl', 'repump_MT_ampl', 'MOT_coil_set', 'pump_AOM_freq', 'repump_AOM_freq', 'OP_AOM_freq', 'pump_ampl', 'repump_ampl', 'OP_ampl', 'pump_reference', 'repump_reference', 'RFtime', 'fmin', 'fmax', 'RFampl', 'RFsweep', 'wait_pump', 'wait_repump', 'load_linfit_1', 'load_linfit_2', 'wait_background', 'test_coil_set', 'test_MOT_coil_set', 'test_pump_AOM_freq', 'test_repump_AOM_freq', 'test_pump_ampl', 'test_repump_ampl', 'test_pump_reference', 'test_repump_reference', 'test_MOT_coil', 'numPowers', 'comp_time', 'repump_time', 'exptime', 'clearbool', 'wallbool', 't_ramp', 'clearfrac', 'precut_fmin', 'precut_t', 'filtertime', 'master_clear', 'tempV', 'currV', 'cat_AOM_freq', 'cat_AOM_ampl', 'cat_deload_t', 'MOT_reload_t', 'timestamp']


columns_with_nan = df[:-2].columns[df[:-2].isna().any()].tolist()

print("Columns with NaN values:", columns_with_nan)

Columns with NaN values: ['betaPAErr']


plt.plot(df['betaPAErr']/df['betaPA'])

[<matplotlib.lines.Line2D at 0x276cfde6ad0>]


plt.plot(df['betaPAErr'][0]/df['betaPA'][0])

[<matplotlib.lines.Line2D at 0x276c10e37d0>]


plt.plot(df['betaPAErr'][:4]/df['betaPA'][:4])

[<matplotlib.lines.Line2D at 0x276c1130e50>]


MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'MegaRuns', 'testPArunMega3')
df = get_data_frame(MEASURE_FOLDER,
                    plot=False,
                    cache_all=True,
                    storeFitResults=False)
dfc= df.copy()
#df.dropna(inplace=True)

groupbyKey = 'pump_reference'
titleKey = 'pump_AOM_freq'

df_grouped = df.groupby(by=groupbyKey)
min_ratios = df_grouped['ratio'].min()

groups = dict(list(df_grouped))
dfs = [df for df in groups.values()]

# plotting ratio vs freq
max_freqs = [384182.5]*len(dfs)
zipped_data = list(zip(dfs, max_freqs))
fig, ax = plt.subplots()
for i, (df, max_freq)  in enumerate(zipped_data[:]):
    data = df
    freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
    ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.0, ms=5, figsize=(10, 10), label=f"{groupbyKey} = { data.iloc[10][groupbyKey] :.2f}", linewidth=2.5)

plt.legend()
plt.savefig(os.path.join(MEASURE_FOLDER, 'lossFeatures.png'))
plt.title(f'Loss Features, {titleKey} = {data[titleKey].mean():.2f}', **titledict)
plt.show()
plt.close()
#---------------------------------------------------
x = [df[groupbyKey].mean() for df in dfs]
y = [(df['ratio'].max() - df['ratio'].min()) for df in dfs]
plt.plot( x, y ,'-o')
plt.xlabel(groupbyKey)
plt.ylabel(r'SNR $ = V_{ss, off} - V_{ss, on}$ ')
plt.title(f'SNR Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict)
plt.show()
plt.savefig(join(MEASURE_FOLDER, 'SNRplot.png'), dpi=200)
plt.close()

  0%|          | 0/211 [00:03<?, ?it/s]

---------------------------------------------------------------------------
RemoteTraceback                           Traceback (most recent call last)
RemoteTraceback: 
"""
Traceback (most recent call last):
  File "c:\ProgramData\Anaconda3\envs\magpy_env\Lib\multiprocessing\pool.py", line 125, in worker
    result = (True, func(*args, **kwds))
                    ^^^^^^^^^^^^^^^^^^^
  File "c:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\analysis\analysis_test.py", line 415, in get_row
    fit_results, settings, timestamp = extract_fit(run_path, **kwargs)
                                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
TypeError: extract_fit() got an unexpected keyword argument 'storeFitResults'
"""

The above exception was the direct cause of the following exception:

TypeError                                 Traceback (most recent call last)
Cell In[39], line 2
      1 MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'MegaRuns', 'testPArunMega3')
----> 2 df = get_data_frame(MEASURE_FOLDER,
      3                     plot=False,
      4                     cache_all=True,
      5                     storeFitResults=False)
      6 dfc= df.copy()
      7 #df.dropna(inplace=True)

Cell In[22], line 438, in get_data_frame(data_dir, parallel, in_process_run, **kwargs)
    436 if parallel:
    437 	with Pool(4) as p:
--> 438 		rows = list(tqdm(p.imap(partial(get_row, **kwargs), run_path_arr), total=len(run_path_arr)))
    440 else:
    441 	for run_path in tqdm(run_path_arr):

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\site-packages\tqdm\std.py:1178, in tqdm.__iter__(self)
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/tqdm/std.py?line=1174'>1175</a> time = self._time
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/tqdm/std.py?line=1176'>1177</a> try:
-> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/tqdm/std.py?line=1177'>1178</a>     for obj in iterable:
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/tqdm/std.py?line=1178'>1179</a>         yield obj
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/tqdm/std.py?line=1179'>1180</a>         # Update and possibly print the progressbar.
   <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/site-packages/tqdm/std.py?line=1180'>1181</a>         # Note: does not call self.update(1) for speed optimisation.

File c:\ProgramData\Anaconda3\envs\magpy_env\Lib\multiprocessing\pool.py:873, in IMapIterator.next(self, timeout)
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/multiprocessing/pool.py?line=870'>871</a> if success:
    <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/multiprocessing/pool.py?line=871'>872</a>     return value
--> <a href='file:///c%3A/ProgramData/Anaconda3/envs/magpy_env/Lib/multiprocessing/pool.py?line=872'>873</a> raise value

TypeError: extract_fit() got an unexpected keyword argument 'storeFitResults'


import os
from os.path import join 
import sys
from functools import partial
sys.path.append(os.path.join(os.getcwd(), '..')) #adds directory below as valid path
from  datetime import datetime, timedelta
dateformat = "%H-%M-%S"
from collections import deque
import traceback
from multiprocessing import Pool
from tqdm import tqdm

import scipy.constants as spc
from lmfit import Model, create_params

from scipy.integrate import odeint
from scipy.optimize import curve_fit

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

plt.rcParams['axes.grid'] = True
plt.rcParams['grid.linestyle'] = '--'

from MT_class_PID_new import MTdataHost
from global_folder.myplotsty import *
from global_folder.my_helpers import *

PUMP_FREQUENCY = 384228.6
REPUMP_FREQUENCY = 384228.6 + 6.56
SAMPLE_RATE = 2000
FREQVSVOLT = 221.0 
FREQVSCURR = 1.13

# TODO: find a better place for this
EXP_FOLDER =r'C:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\CATMeasurements'
MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'testPArun16')
WDATA_FOLDER =os.path.join(MEASURE_FOLDER, 'testPArun16.csv')

# TODO: maybe make a run analysis class out of this?
def dump():

	collect_plots(MEASURE_FOLDER, os.path.join(MEASURE_FOLDER, 'collected_plots'), 'deloadPhase.png')
 
	#*-----------------------
	#* SINGLE RUN
 	#*----------------------- 
  
	MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'testPArun9')
	df = get_data_frame(MEASURE_FOLDER)
	df.dropna(inplace=True)
	#freqs = plot_results(df, 384201., save_folder=MEASURE_FOLDER)
 
	max_freq = 384182.5
	data = df
 
	freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-df['tempV'].min())*FREQVSVOLT- (data['currV']-df['currV'].min())*FREQVSCURR)
	fig, ax = plt.subplots()
	plot_spline_fit(ax=ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'], 
				 s=0.0, save_folder=MEASURE_FOLDER, 
				 mfc='red', color='black', 
				 title='')
	plt.show()
	plt.close()
 
	betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
	freqs = sorted(freqs)
	plt.plot(freqs, betaPAs, 'o-', ms=5, label="")
	plt.legend()
	plt.xlabel(r'$\Delta$ (GHz)')
	plt.ylabel(r'$\beta_{\mathrm{eff}}$ ')
	#plt.savefig(join(MEASURE_FOLDER, 'betaVsFreq.png'), dpi=200) 
	plt.title(f"2-body Decay Plot {''} ", **titledict) 
	plt.show()
	plt.close()
 
 	# *-----------------------
	# * MULTIPLE RUN COMPARISON
 	# *-----------------------
	folders = [os.path.join(EXP_FOLDER, path ) for path in ['PArunHalfVarDet1', 'testPAVaryingCATampl']]
	dfs = [get_data_frame(measure_folder) for measure_folder in folders]

	dfs = [get_data_frame(measure_folder) for measure_folder in folders]

	max_freqs = [384182.6]*len(dfs)
	zipped_data = list(zip(dfs, max_freqs))
	for i, (df, max_freq)  in enumerate(zipped_data[:]):
		fig, ax = plt.subplots()
		data = df.dropna()
		freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'], scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.00, ms=5, save_folder=join(folders[i]))

	#*-----------------------
	#* PARSING WAVEMETER DATA
 	#*-----------------------

	MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'testPArun14')
	WDATA_FOLDER =os.path.join(MEASURE_FOLDER, 'testPArun14.csv')
	freq_data, max_freq, min_freq = add_wavemeter_data('', WDATA_FOLDER)
	data = freq_data[:]
	levels = staircase_fit(data)

	data = get_data_frame(MEASURE_FOLDER)
	data.dropna(inplace=True)
	freqs = ((max_freq)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
	plt.plot(freqs)
 


	#*-----------------------
	#* MEGA_RUN
 	#*-----------------------	
	MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'MegaRuns', 'testPArunMega3')
	df = get_data_frame(MEASURE_FOLDER,
						plot=False,
						cache_all=True)
	dfc= df.copy()
	#df.dropna(inplace=True)

	groupbyKey = 'pump_reference'
	titleKey = 'pump_AOM_freq'

	df_grouped = df.groupby(by=groupbyKey)
	min_ratios = df_grouped['ratio'].min()

	groups = dict(list(df_grouped))
	dfs = [df for df in groups.values()]

	# plotting ratio vs freq
	max_freqs = [384182.5]*len(dfs)
	zipped_data = list(zip(dfs, max_freqs))
	fig, ax = plt.subplots()
	for i, (df, max_freq)  in enumerate(zipped_data[:]):
		data = df
		freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.0, ms=5, figsize=(10, 10), label=f"{groupbyKey} = { data.iloc[10][groupbyKey] :.2f}", linewidth=2.5)

	plt.legend()
	plt.savefig(os.path.join(MEASURE_FOLDER, 'lossFeatures.png'))
	plt.title(f'Loss Features, {titleKey} = {data[titleKey].mean():.2f}', **titledict)
	plt.show()
	plt.close()
	#---------------------------------------------------
	x = [df[groupbyKey].mean() for df in dfs]
	y = [(df['ratio'].max() - df['ratio'].min()) for df in dfs]
	plt.plot( x, y ,'-o')
	plt.xlabel(groupbyKey)
	plt.ylabel(r'SNR $ = V_{ss, off} - V_{ss, on}$ ')
	plt.title(f'SNR Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict)
	plt.show()
	plt.savefig(join(MEASURE_FOLDER, 'SNRplot.png'), dpi=200)
	plt.close()

	#---------------------------------------------------
	# Plotting 2 body decay rate
	#dfs = [df for df in groups.values()]

	for i, df  in enumerate(dfs[:]):
		data = df
		freqs = ((max_freqs[0]-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
		freqs = sorted(freqs)
		plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{groupbyKey}={data.iloc[10][groupbyKey]:.2f}")
	plt.legend()
	plt.xlabel(r'$\Delta$ (GHz)')
	plt.ylabel(r'$\beta_{\mathrm{eff}}$ ')
	plt.savefig(join(MEASURE_FOLDER, 'betaVsFreq.png'), dpi=200) 
	plt.title(f'2-body Decay Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict) 
	plt.show()
	plt.close()
	#*-----------------------
	#* MULTIPLE MEGARUN
	#*-----------------------
 
	folders = [os.path.join(EXP_FOLDER, 'MegaRuns', path ) for path in ['testPArunMega7', 'testPArunMega8']]

	dfs_mega = [get_data_frame(measure_folder, cache_all=True) for measure_folder in folders]

	groupbyKey = 'pump_reference'
	titleKey = 'pump_AOM_freq'

	dfs_grouped = [df_mega.groupby(by=groupbyKey) for df_mega in dfs_mega]
	min_ratios = [df_grouped['ratio'].min() for df_grouped in dfs_grouped]

	groupss = [dict(list(df_grouped)) for df_grouped in dfs_grouped]
	dfs = [ [df for df in groups.values()] for groups in groupss]
 
	for row in dfs:
		data = row[3]
	
		freqs = ((384182.5-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
		freqs = sorted(freqs)
		plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{titleKey}={data.iloc[10][titleKey]:.2f}")
		plt.title(f'{groupbyKey} = {data.iloc[10][groupbyKey]:.2f}')
	plt.legend()
	
	#*-----------------------
	#* FULL RUNS
	#*-----------------------
	
	MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'testPArunFull4')
	df = get_data_frame(MEASURE_FOLDER)
	df.dropna(inplace=True)

	dfc = df.copy()
	df_grouped = df.groupby(by=['pump_reference', 'pump_AOM_freq'])
	groups = dict(list(df_grouped))
	dfs = [df for df in groups.values()]
 
	max_freqs = [384182.8]*30
	zipped_data = list(zip(dfs, max_freqs))
	fig1, ax1s = plt.subplots(4)
	fig2, ax2s = plt.subplots(5)
	for i, (df, max_freq)  in enumerate(zipped_data[:]):
		j1 = i%4
		j2 = i//4
		
		data = df.dropna()
		freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-0.0)*FREQVSCURR)
		ax2s[j2] = plot_spline_fit(ax2s[j2], x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i%4}', mfc=f'C{i%4}',color=f'C{i%4}', s=0.0, ms=5, figsize=(5, 25), linewidth=1.5, label=f"Detuning  = { 180-2*df.iloc[10]['pump_AOM_freq'] :.2f}", fig=fig2)
		ax2s[j2].set_title(f"Pump Amplituide = { df.iloc[10]['pump_reference'] :.2f}", **titledict)
		ax2s[j2].legend()
		
		ax1s[j1] = plot_spline_fit(ax1s[j1], x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i//4}', mfc=f'C{i//4}',color=f'C{i//4}', s=0.0, ms=5, figsize=(5, 25), linewidth=1.5, label=f"Pump Amplitude = { df.iloc[10]['pump_reference'] :.2f}", fig=fig1)
		ax1s[j1].set_title(f"Detuning  = { 180-2*df.iloc[10]['pump_AOM_freq'] :.2f}", **titledict)
		ax1s[j1].legend()
		
	fig1.tight_layout()
	fig2.tight_layout()

	fig1.savefig(os.path.join(MEASURE_FOLDER, 'lossFeaturesDet.png'))
	plt.show()
	plt.close()

	fig2.savefig(os.path.join(MEASURE_FOLDER, 'lossFeaturesPA.png'))
	plt.show()
	plt.close()
	
	SNRdata = df_grouped['ratio'].max() - df_grouped['ratio'].min()
	SNRdf = SNRdata.reset_index()
	SNRdf.columns = ['pump_reference', 'pump_AOM_freq', 'SNR']
	pivot_table = SNRdf.pivot('pump_reference', 'pump_AOM_freq', 'SNR')
	xticklabels = [f'{180-2*x:.2f}' for x in pivot_table.columns]
	yticklabels = [f'{y:.2f}' for y in pivot_table.index]
	sns.heatmap(pivot_table, annot=True, fmt='.2f', xticklabels=xticklabels, yticklabels=yticklabels)
	plt.xlabel("Detuning (MHz)")
	plt.ylabel("Pump Reference")
	plt.grid()
	plt.savefig(os.path.join(MEASURE_FOLDER, 'heatmap.png'))
	plt.show()
	plt.close()

	# fig, ax = plt.subplots()
	# for i, (df, max_freq)  in enumerate(zipped_data[:]):
	# 	data = df.dropna()
	# 	freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		
	# 	ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.0, ms=5, figsize=(10, 10), linewidth=2.5)
		
	# 	plt.title(f"Pump Amplituide = { df.iloc[10]['pump_reference'] :.2f}, \
	# 				sDetuning  = { 180-2*df.iloc[10]['pump_AOM_freq'] :.2f}", **titledict)

	# 	plt.legend()

	# 	plt.savefig(os.path.join(MEASURE_FOLDER, f'lossFeatures{i}.png'))
	# 	plt.show()
		
	# 	plt.close()
	# 	fig, ax = plt.subplots()



def freq_misc():
	WDATA_FOLDER = r'C:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\CATMeasurements\CATcurrTestrun3.csv'
	freq_data = add_wavemeter_data('', WDATA_FOLDER)
	levels = staircase_fit(freq_data[0], peak_height=0.2, distance=50, data_offset=1, window_size=1) 

	plt.close()
 
	x = np.linspace(0, 4.9, 25)
	y = levels
 
	plt.plot(levels, '-o')
	plt.title('Levels plot')
	
	m, b, fit_line = my_linear_fit(x, y)
def save_fit_results(run_path, plot=False, bkfile=False, 
					 CATbaseline=True, MOTbaseline=True,
					 initRFit=True, loadFit=True,deloadFit=True,reloadFit=True,
					 storeFitResults=True):
	filename = os.path.join(run_path, 'data.csv')
	bkfilename = os.path.join(run_path, 'data_oldPD.csv')
	settingsname = os.path.join(run_path, 'Settings.txt')
	
	dataHost = MTdataHost(SAMPLE_RATE)
	dataHost.loadCATdata(fileName=filename, settingsName=settingsname)
	
	if bkfile:
		dataHost.CATbackgroundData(bkfilename)
	
	#dataHost.setAllCAT(0.002)
	if CATbaseline:
		dataHost.setCATbaseline(0.002)
	if MOTbaseline:
		dataHost.setBaseline(0.002)
	if loadFit:
		dataHost.setLoading(0.002)
	if initRFit and loadFit:
		dataHost.initFit, dataHost.initX = dataHost.setInitialLoad(0.002)
	if deloadFit:
		dataHost.setDeloading(0.002)
		dataHost.plotDeloadFit(run_path)# TODO: currently just stores the deloading times and voltages
	if reloadFit:
		dataHost.setReloadVolt(0.002)

	if CATbaseline and MOTbaseline and loadFit and reloadFit:
	# steady state ratio fraction
		dataHost.ratio = dataHost.reloadVolt / dataHost.motSS

		dataHost.ratioErr = dataHost.ratio * ((dataHost.reloadVoltErr/dataHost.reloadVolt)**2 + (dataHost.motSSErr/dataHost.motSS)**2)**(0.5)
	
		# if abs(dataHost.ratioErr / dataHost.ratio) > 0.1:
		#     dataHost.ratioErr = abs(0.015*dataHost.ratio)
		if dataHost.ratioErr < 0.001:
			dataHost.ratioErr = 0.001
		if dataHost.ratio < 0:
			dataHost.ratio = 0
			
		# TODO: this information is useless
	print('File loaded: RFmin = {} MHz, t_mt = {:.3f} s.'.format(dataHost.settings['fmin'], dataHost.settings['wait_mtrap']))
	
	resultDict = dataHost.getResults(run_path, store=storeFitResults)
 
	if plot:
		dataHost.storeFits(run_path, combined=True, separate=True)
  
	return resultDict, dataHost.settings
		
def get_timestamp(run_path):
	timestamp = datetime.strptime(os.path.split(run_path)[-1].split('_')[0], dateformat)
	return timestamp

def extract_fit(run_path, plot=True, cache_failed=True, cache_all=True, **kwargs):
	"""Gather relevant data from each measurement run

	Args:
		run_path : absolute path to the run directory
		plot (bool, optional): plot fits. Defaults to True.
		cache_failed (bool, optional): Cache failed fits. If false, refit. Doesn't refit non-failed fits. Defaults to True.
		cache_all (bool, optional): If false, ignore any cached fit_results. Defaults to True.

	Returns:
		a 3-tuple (fit_results, settings, timestamp)
	"""
	fit_results, settings, timestamp = {}, {}, None
	
	if not os.path.isdir(run_path):
		return fit_results, settings, timestamp  # directory is not a run directory

	try:
		timestamp = get_timestamp(run_path)
  
  	# TODO: specify which error to catch
	except Exception as e:
		print("Error extracting timestamp from: ", run_path)
		print(traceback.format_exc())
	
	MAT_fit_cache_path = os.path.join(run_path, 'resultDict.txt')
 
	if not os.path.exists(MAT_fit_cache_path) or not cache_all:
		
		try: 
			fit_results, settings = save_fit_results(run_path, plot=plot, **kwargs)
		except Exception as e:
			print(traceback.format_exc())
			print("Fitting ERROR at ", os.path.basename(run_path), '\n')

			with open(MAT_fit_cache_path, 'w') as f:
				f.write(str('MAT fit failed'))
		
	else:
		print("Accessing cached results from :", os.path.basename(run_path))
  
		fit_results = open(MAT_fit_cache_path, 'r').read()

		if fit_results == 'MAT fit failed':
			if not cache_failed:
				# fit regardless of cached result
				try: 
					fit_results, settings = save_fit_results(run_path, plot=plot)
				except Exception as e:
					print(traceback.format_exc())
					print("Fitting ERROR at ", os.path.basename(run_path), '\n')
	
					with open(MAT_fit_cache_path, 'w') as f:
						f.write(str('MAT fit failed'))	
			else:
				print("Failed fit at :", os.path.basename(run_path))
				fit_results = {}

		else:
			fit_results = eval(open(MAT_fit_cache_path, 'r').read())
   
			settingsname = os.path.join(run_path, 'Settings.txt')
			settings = eval(open(settingsname, 'r').read())
   
	return fit_results, settings, timestamp


def get_row(run_path, **kwargs):
			
   fit_results, settings, timestamp = extract_fit(run_path, **kwargs)
   row = {**fit_results, **settings, **{'timestamp':timestamp}}
   
   return row

   
def get_data_frame(data_dir, parallel=True, in_process_run=False, **kwargs):
   
	run_path_arr = []
	rows = []
	
	for relative_path in os.listdir(data_dir):
		run_path_arr.append(os.path.join(data_dir, relative_path))
  
	if in_process_run:
		run_path_arr.pop()

	run_path_arr = sorted(run_path_arr)
	
	if parallel:
		with Pool(4) as p:
			rows = list(tqdm(p.imap(partial(get_row, **kwargs), run_path_arr), total=len(run_path_arr)))
	
	else:
		for run_path in tqdm(run_path_arr):
		   rows.append(get_row(run_path, **kwargs))
	return pd.DataFrame.from_dict(rows)

def add_wavemeter_data(df, wmeter_csv_path, window_size=100, num_rows=50):


	"""Extract unique frequnecy values from wavemeter data
	Returns:
		unique_levels (list): unique frequency values in wavemeter data
	"""
 
	# TODO: modify dataframe in place with frequency data
 
	wdata = pd.read_csv(wmeter_csv_path, skiprows=2)
	wdata.dropna(inplace=True)
	freq_data = np.array(wdata.iloc[:, 0])
	try:
		freq_data = np.array([float(item) for item in freq_data if item.replace('.','').isdigit()])
	except Exception as e:
		print(e)
 
	max_freq = freq_data.max()
	min_freq = freq_data.min()

	return freq_data, max_freq, min_freq

def plot_results(ax, dfs, max_freq, min_freq=0.0, mfc='red', fmt='o', ms=5, save_folder=False, xscale=1.0, yscale=1.0, **kwargs):
	FREQVSVOLT = 221.0 
	FREQVSCURR = 1.13
	
	if not type(dfs) == list:
		freqs = ((max_freq-PUMP_FREQUENCY)-(dfs.dropna()['tempV']-dfs.dropna()['tempV'].min())*FREQVSVOLT- (dfs.dropna()['currV']-dfs.dropna()['currV'].min())*FREQVSCURR)*xscale
		dfs=[dfs]
  
	plt.gcf().set_dpi(300)
	
	for df in dfs:
		df = df.dropna()
		ax.errorbar(freqs, 
			   df['ratio']*yscale,
			   yerr=df['ratioErr'],
			   fmt=fmt, mfc=mfc, color='black', ms=ms, **kwargs)
		ax.set_ylabel(r'$\mathbf{\frac{V_{ss, cat}}{V_{ss}}} $ ', **labeldict)
		ax.set_xlabel(r'$\Delta $ (GHz)', **labeldict)
	
	if save_folder:
		plt.savefig(os.path.join(save_folder, 'ratio_vs_freq.png'))
	
	return freqs, ax
  
	#return plt.gca(), plt.gcf()
	#plt.show()

def plot_spline_fit(ax, x, y, s=1, yerr=None, color='black', scolor='black',figsize=(12,5), save_folder=None, title='',alpha=0.5,dpi=200, label='plot', fig=None,**kwargs):
	from scipy.interpolate import splev, splrep
	xnew = np.linspace(min(x), max(x), 3*len(x) )

	y = [b for a,b in sorted(zip(x,y), key=lambda pair: pair[0])]
	if yerr is not None:
		yerr = [b for a,b in sorted(zip(x,yerr), key=lambda pair: pair[0])]
 
	x = sorted(x)


	spl = splrep(x, y, s=s)
	ynew = splev(xnew, spl)

	if fig is None:
		plt.gcf().set_dpi(dpi)
		plt.gcf().set_size_inches(figsize)
	else:
		fig.set_dpi(dpi)
		fig.set_size_inches(figsize)
	if yerr is not None:
		ax.errorbar(x, y, yerr=yerr, fmt='o', color=color, **kwargs)
	else:
		ax.plot(x,y, 'o', **kwargs)
	ax.plot(xnew, ynew, '-', color=scolor, alpha=alpha, label=label, **kwargs)
 
	ax.set_ylabel(r'$\mathbf{\frac{V_{ss, cat}}{V_{ss}}} $ ', **labeldict)
	ax.set_xlabel(r'$\Delta $ (GHz)', **labeldict)
 
	ax.set_title(title, **titledict)
	if save_folder:
		plt.savefig(os.path.join(save_folder, 'spline_ratio_vs_freq.png'))
	
	return ax
 
def plot_polyfit(x_data, y_data, spline_degree):
	
	coefficients = np.polyfit(x_data, y_data, spline_degree)

	x_interp = np.linspace(min(x_data), max(x_data), 100)
	y_interp = np.polyval(coefficients, x_interp)

	plt.scatter(x_data, y_data, label='Original Data')
	plt.plot(x_interp, y_interp, label='Polynomial Interpolation (Degree={})'.format(spline_degree))

def collect_plots(source, destination, plot_name):
	print(f'Collecting plots from {os.path.basename(source)}')
	import shutil

	os.makedirs(destination, exist_ok=True)

	plot_files = []
	for root, dirs, files in os.walk(source):
		
		for file in files:
			if file == plot_name:
				plot_files.append(os.path.join(root, file))

	for i, plot_file in enumerate(plot_files, start=0):
		new_filename = f'{i}{plot_name}'
		destination_path = os.path.join(destination, new_filename)
		shutil.copy(plot_file, destination_path)

def create_GIF(images_folder, image_name):
	import imageio
	with imageio.get_writer(os.path.join(images_folder, f'{image_name}movie.gif'), mode='I', duration=0.5) as writer:
		for filename in os.listdir(images_folder):
			if image_name in filename:
				image = imageio.imread(os.path.join(images_folder, filename))
				writer.append_data(image)

def staircase_fit(data, peak_height=0.1, distance=100, data_offset=1, window_size=1, inc_final_peak=True):
	def moving_average(arr, window_size):
		weights = np.ones(window_size) / window_size
		return np.convolve(arr, weights, mode='valid')

	convdata1 = moving_average(data, window_size)
	convdata2 = moving_average(data[data_offset+1:], window_size )
	final = convdata1[:len(convdata2)]-convdata2
	# plt.plot(data)
	# plt.plot(convdata1)
	# plt.plot(convdata2)
	# plt.show()
	# plt.plot(final)

	
	from scipy.signal import find_peaks
	x= final
	peaks, _ = find_peaks(x, height=peak_height, distance=distance)
	peaks = np.insert(peaks, 0, 0)
	levels = []
	plot_arr = []
	for i, peak in enumerate(peaks):
		
		if i < len(peaks) - 1:
			temp = data[ peaks[i]:peaks[i+1] ]
			plot_arr.extend( np.ones_like(temp)*np.mean(temp))
			levels.append(np.mean(temp))
	if inc_final_peak:
		temp = data[peaks[-1]:]
		levels.append(np.mean(temp))
		plot_arr.extend( np.ones_like(temp)*np.mean(temp))
  
	plt.plot(x)
	plt.plot(peaks, x[peaks], "x")
	plt.plot(np.zeros_like(x), "--", color="gray")
	plt.show()
	
	plt.plot(data)
	plt.plot(np.ravel((plot_arr)))
	plt.show()
	plt.close()
	plt.plot(np.array(levels)[np.where(abs(np.diff(levels))>0.05)[0]], 'o', ms=5)
 
	return levels

def load_single_run(run_path):
	filename = os.path.join(run_path, 'data.csv')
	bkfilename = os.path.join(run_path, 'data_oldPD.csv')
	settingsname = os.path.join(run_path, 'Settings.txt')
	
	dh1 = MTdataHost(SAMPLE_RATE)
	dh1.loadCATdata(fileName=filename, settingsName=settingsname)
	
	return dh1
	
def load_mega_run(MEASURE_FOLDER, groupbyKey, titleKey, plot=True, save_plots=False, max_freq=384182.5, **kwargs):
    # TODO: use the plot flag to do something?
    
	df = get_data_frame(MEASURE_FOLDER,
						**kwargs)
	dfc= df.copy()
	#df.dropna(inplace=True)

	df_grouped = df.groupby(by=groupbyKey)
	min_ratios = df_grouped['ratio'].min()

	groups = dict(list(df_grouped))
	dfs = [df for df in groups.values()]

	# plotting ratio vs freq
	max_freqs = [max_freq]*len(dfs)
	zipped_data = list(zip(dfs, max_freqs))
	fig1, ax = plt.subplots()
	for i, (df, max_freq)  in enumerate(zipped_data[:]):
		data = df
		freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.0, ms=5, figsize=(10, 10), label=f"{groupbyKey} = { data.iloc[10][groupbyKey] :.2f}", linewidth=2.5)


	plt.legend()
	
	plt.title(f'Loss Features, {titleKey} = {data[titleKey].mean():.2f}', **titledict)
	plt.show()

	#---------------------------------------------------
	fig2 = plt.figure(2)
	x = [df[groupbyKey].mean() for df in dfs]
	y = [(df['ratio'].max() - df['ratio'].min()) for df in dfs]
	plt.plot( x, y ,'-o')
	plt.xlabel(groupbyKey)
	plt.ylabel(r'SNR $ = V_{ss, off} - V_{ss, on}$ ')
	plt.title(f'SNR Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict)
	plt.show()
	
	#---------------------------------------------------------
	fig3=plt.figure(3)
	for i, df  in enumerate(dfs[:]):
		data = df
		freqs = ((384182.5-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
		betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
		freqs = sorted(freqs)
		plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{groupbyKey}={data.iloc[10][groupbyKey]:.2f}")
	plt.legend()
	plt.xlabel(r'$\Delta$ (GHz)')
	plt.ylabel(r'$\beta_{\mathrm{eff}}$ ')
	plt.title(f'2-body Decay Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict) 
	plt.show()

	if save_plots:
		fig1.savefig(os.path.join(MEASURE_FOLDER, 'lossFeatures.png'))
		fig2.savefig(join(MEASURE_FOLDER, 'SNRplot.png'), dpi=200)
		fig3.savefig(join(MEASURE_FOLDER, 'betaVsFreq.png'), dpi=200) 
	return dfc


if __name__ == '__main__':
	# run_path = r"C:\Users\svars\OneDrive\Desktop\UBC Lab\CATExperiment\CATMeasurements\testPArun9\16-53-10"
	# filename = os.path.join(run_path, 'data.csv')
	# bkfilename = os.path.join(run_path, 'data_oldPD.csv')
	# settingsname = os.path.join(run_path, 'Settings.txt')
	
	# dh1 = MTdataHost(SAMPLE_RATE)
	# dh1.loadCATdata(fileName=filename, settingsName=settingsname)
	
	# dh1.setAllCAT(0.002)
 
 	#dh1.CATbackgroundData(bkfilename)
	
	pass


MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'MegaRuns', 'testPArunMega3')
df = get_data_frame(MEASURE_FOLDER,
                    plot=False,
                    cache_all=True,
                    storeFitResults=False)
dfc= df.copy()
#df.dropna(inplace=True)

groupbyKey = 'pump_reference'
titleKey = 'pump_AOM_freq'

df_grouped = df.groupby(by=groupbyKey)
min_ratios = df_grouped['ratio'].min()

groups = dict(list(df_grouped))
dfs = [df for df in groups.values()]

# plotting ratio vs freq
max_freqs = [384182.5]*len(dfs)
zipped_data = list(zip(dfs, max_freqs))
fig, ax = plt.subplots()
for i, (df, max_freq)  in enumerate(zipped_data[:]):
    data = df
    freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
    ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.0, ms=5, figsize=(10, 10), label=f"{groupbyKey} = { data.iloc[10][groupbyKey] :.2f}", linewidth=2.5)

plt.legend()
plt.savefig(os.path.join(MEASURE_FOLDER, 'lossFeatures.png'))
plt.title(f'Loss Features, {titleKey} = {data[titleKey].mean():.2f}', **titledict)
plt.show()
plt.close()
#---------------------------------------------------
x = [df[groupbyKey].mean() for df in dfs]
y = [(df['ratio'].max() - df['ratio'].min()) for df in dfs]
plt.plot( x, y ,'-o')
plt.xlabel(groupbyKey)
plt.ylabel(r'SNR $ = V_{ss, off} - V_{ss, on}$ ')
plt.title(f'SNR Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict)
plt.show()
plt.savefig(join(MEASURE_FOLDER, 'SNRplot.png'), dpi=200)
plt.close()

100%|██████████| 211/211 [00:03<00:00, 63.09it/s]


MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'MegaRuns', 'testPArunMega3')
df = get_data_frame(MEASURE_FOLDER,
                plot=False,
                cache_all=False,
                storeFitResults=False)
dfc= df.copy()
#df.dropna(inplace=True)

groupbyKey = 'pump_reference'
titleKey = 'pump_AOM_freq'

df_grouped = df.groupby(by=groupbyKey)
min_ratios = df_grouped['ratio'].min()

groups = dict(list(df_grouped))
dfs = [df for df in groups.values()]

# plotting ratio vs freq
max_freqs = [384182.5]*len(dfs)
zipped_data = list(zip(dfs, max_freqs))
fig, ax = plt.subplots()
for i, (df, max_freq)  in enumerate(zipped_data[:]):
data = df
freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.0, ms=5, figsize=(10, 10), label=f"{groupbyKey} = { data.iloc[10][groupbyKey] :.2f}", linewidth=2.5)

plt.legend()
#plt.savefig(os.path.join(MEASURE_FOLDER, 'lossFeatures.png'))
plt.title(f'Loss Features, {titleKey} = {data[titleKey].mean():.2f}', **titledict)
plt.show()
plt.close()
#---------------------------------------------------
x = [df[groupbyKey].mean() for df in dfs]
y = [(df['ratio'].max() - df['ratio'].min()) for df in dfs]
plt.plot( x, y ,'-o')
plt.xlabel(groupbyKey)
plt.ylabel(r'SNR $ = V_{ss, off} - V_{ss, on}$ ')
plt.title(f'SNR Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict)
plt.show()
#plt.savefig(join(MEASURE_FOLDER, 'SNRplot.png'), dpi=200)
plt.close()

x = [df[groupbyKey].mean() for df in dfs]
y = [(df['ratio'].max() - df['ratio'].min()) for df in dfs]
plt.plot( x, y ,'-o')
plt.xlabel(groupbyKey)
plt.ylabel(r'SNR $ = V_{ss, off} - V_{ss, on}$ ')
plt.title(f'SNR Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict)
plt.show()
#plt.savefig(join(MEASURE_FOLDER, 'SNRplot.png'), dpi=200)
plt.close()

  Cell In[42], line 23
    data = df
    ^
IndentationError: expected an indented block after 'for' statement on line 22


MEASURE_FOLDER = os.path.join(EXP_FOLDER, 'MegaRuns', 'testPArunMega3')
df = get_data_frame(MEASURE_FOLDER,
                plot=False,
                cache_all=False,
                storeFitResults=False)
dfc= df.copy()
#df.dropna(inplace=True)

groupbyKey = 'pump_reference'
titleKey = 'pump_AOM_freq'

df_grouped = df.groupby(by=groupbyKey)
min_ratios = df_grouped['ratio'].min()

groups = dict(list(df_grouped))
dfs = [df for df in groups.values()]

# plotting ratio vs freq
max_freqs = [384182.5]*len(dfs)
zipped_data = list(zip(dfs, max_freqs))
fig, ax = plt.subplots()
for i, (df, max_freq)  in enumerate(zipped_data[:]):
    data = df
    freqs = ((max_freq-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
    ax=plot_spline_fit(ax, x=freqs, y=data['ratio'], yerr=data['ratioErr'],scolor=f'C{i}', mfc=f'C{i}',color=f'C{i}', s=0.0, ms=5, figsize=(10, 10), label=f"{groupbyKey} = { data.iloc[10][groupbyKey] :.2f}", linewidth=2.5)

plt.legend()
#plt.savefig(os.path.join(MEASURE_FOLDER, 'lossFeatures.png'))
plt.title(f'Loss Features, {titleKey} = {data[titleKey].mean():.2f}', **titledict)
plt.show()
plt.close()
#---------------------------------------------------
x = [df[groupbyKey].mean() for df in dfs]
y = [(df['ratio'].max() - df['ratio'].min()) for df in dfs]
plt.plot( x, y ,'-o')
plt.xlabel(groupbyKey)
plt.ylabel(r'SNR $ = V_{ss, off} - V_{ss, on}$ ')
plt.title(f'SNR Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict)
plt.show()
#plt.savefig(join(MEASURE_FOLDER, 'SNRplot.png'), dpi=200)
plt.close()

x = [df[groupbyKey].mean() for df in dfs]
y = [(df['ratio'].max() - df['ratio'].min()) for df in dfs]
plt.plot( x, y ,'-o')
plt.xlabel(groupbyKey)
plt.ylabel(r'SNR $ = V_{ss, off} - V_{ss, on}$ ')
plt.title(f'SNR Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict)
plt.show()
#plt.savefig(join(MEASURE_FOLDER, 'SNRplot.png'), dpi=200)
plt.close()

100%|██████████| 211/211 [02:45<00:00,  1.27it/s]


for i, df  in enumerate(dfs[:]):
    data = df
    freqs = ((max_freqs[0]-PUMP_FREQUENCY)-(data['tempV']-data['tempV'].min())*FREQVSVOLT- (data['currV']-data['currV'].min())*FREQVSCURR)
    betaPAs = [a for a,b in sorted(zip(data['betaPA'], freqs), key=lambda pair:pair[1])]
    freqs = sorted(freqs)
    plt.plot(freqs, betaPAs, 'o-', ms=5, label=f"{groupbyKey}={data.iloc[10][groupbyKey]:.2f}")
plt.legend()
plt.xlabel(r'$\Delta$ (GHz)')
plt.ylabel(r'$\beta_{\mathrm{eff}}$ ')
#plt.savefig(join(MEASURE_FOLDER, 'betaVsFreq.png'), dpi=200) 
plt.title(f'2-body Decay Plot, {titleKey} = {df[titleKey].mean():.2f}', **titledict) 
plt.show()
plt.close()

	sampleRate	extraTime	timeHold	timeBaseline	timeTest	timeLoad	timeF1	offset	baseVolt	BaseVoltErr	...	precut_t	filtertime	master_clear	tempV	currV	cat_AOM_freq	cat_AOM_ampl	cat_deload_t	MOT_reload_t	timestamp
0	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0400	0.165755	0.000039	...	0.5	0.275	False	0.415224	2.3275	90.0	0.9	30.0	5.0	1900-01-01 16:04:31
1	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0425	0.165896	0.000039	...	0.5	0.275	False	0.415224	1.7150	90.0	0.9	30.0	5.0	1900-01-01 16:06:32
2	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0360	0.166137	0.000039	...	0.5	0.275	False	0.415224	1.8375	90.0	0.9	30.0	5.0	1900-01-01 16:08:36
3	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0140	0.166159	0.000039	...	0.5	0.275	False	0.415224	3.5525	90.0	0.9	30.0	5.0	1900-01-01 16:10:34
4	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0580	0.166390	0.000039	...	0.5	0.275	False	0.415224	2.4500	90.0	0.9	30.0	5.0	1900-01-01 16:12:42
5	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0600	0.166738	0.000038	...	0.5	0.275	False	0.415224	2.5725	90.0	0.9	30.0	5.0	1900-01-01 16:14:57
6	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0200	0.165804	0.000039	...	0.5	0.275	False	0.415224	1.4700	90.0	0.9	30.0	5.0	1900-01-01 16:17:15
7	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0340	0.166250	0.000039	...	0.5	0.275	False	0.415224	2.2050	90.0	0.9	30.0	5.0	1900-01-01 16:19:30
8	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0100	0.165623	0.000039	...	0.5	0.275	False	0.415224	2.6950	90.0	0.9	30.0	5.0	1900-01-01 16:21:32
9	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0295	0.166040	0.000039	...	0.5	0.275	False	0.415224	1.9600	90.0	0.9	30.0	5.0	1900-01-01 16:23:23
10	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0540	0.166372	0.000039	...	0.5	0.275	False	0.415224	3.1850	90.0	0.9	30.0	5.0	1900-01-01 16:25:12
11	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0195	0.166743	0.000038	...	0.5	0.275	False	0.415224	3.3075	90.0	0.9	30.0	5.0	1900-01-01 16:26:59
12	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0085	0.166322	0.000039	...	0.5	0.275	False	0.415224	1.5925	90.0	0.9	30.0	5.0	1900-01-01 16:28:43
13	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0710	0.166550	0.000039	...	0.5	0.275	False	0.415224	3.7975	90.0	0.9	30.0	5.0	1900-01-01 16:30:27
14	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0265	0.166740	0.000038	...	0.5	0.275	False	0.415224	3.6750	90.0	0.9	30.0	5.0	1900-01-01 16:32:09
15	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0715	0.166051	0.000039	...	0.5	0.275	False	0.415224	2.0825	90.0	0.9	30.0	5.0	1900-01-01 16:33:49
16	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0500	0.166400	0.000039	...	0.5	0.275	False	0.415224	3.0625	90.0	0.9	30.0	5.0	1900-01-01 16:35:30
17	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0620	0.166075	0.000039	...	0.5	0.275	False	0.415224	3.4300	90.0	0.9	30.0	5.0	1900-01-01 16:37:11
18	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0155	0.165926	0.000040	...	0.5	0.275	False	0.415224	2.9400	90.0	0.9	30.0	5.0	1900-01-01 16:38:52
19	2000.0	0.0	0.0	1.0	1.0	40.0	0.0	1.0610	0.165490	0.000039	...	0.5	0.275	False	0.415224	2.8175	90.0	0.9	30.0	5.0	1900-01-01 16:40:33
20	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	...	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaT
21	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	...	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaN	NaT

	sampleRate	timeBaseline	timeTest	timeLoad	offset	baseVolt	BaseVoltErr	...	precut_t	filtertime	master_clear	tempV	currV	cat_AOM_freq	cat_AOM_ampl	cat_deload_t	MOT_reload_t	timestamp
0	2000.0	1.0	1.0	40.0	1.0400	0.165755	0.000039	...	0.5	0.275	False	0.415224	2.3275	90.0	0.9	30.0	5.0	1900-01-01 16:04:31
1	2000.0	1.0	1.0	40.0	1.0425	0.165896	0.000039	...	0.5	0.275	False	0.415224	1.7150	90.0	0.9	30.0	5.0	1900-01-01 16:06:32
2	2000.0	1.0	1.0	40.0	1.0360	0.166137	0.000039	...	0.5	0.275	False	0.415224	1.8375	90.0	0.9	30.0	5.0	1900-01-01 16:08:36
3	2000.0	1.0	1.0	40.0	1.0140	0.166159	0.000039	...	0.5	0.275	False	0.415224	3.5525	90.0	0.9	30.0	5.0	1900-01-01 16:10:34
4	2000.0	1.0	1.0	40.0	1.0580	0.166390	0.000039	...	0.5	0.275	False	0.415224	2.4500	90.0	0.9	30.0	5.0	1900-01-01 16:12:42
5	2000.0	1.0	1.0	40.0	1.0600	0.166738	0.000038	...	0.5	0.275	False	0.415224	2.5725	90.0	0.9	30.0	5.0	1900-01-01 16:14:57
6	2000.0	1.0	1.0	40.0	1.0200	0.165804	0.000039	...	0.5	0.275	False	0.415224	1.4700	90.0	0.9	30.0	5.0	1900-01-01 16:17:15
7	2000.0	1.0	1.0	40.0	1.0340	0.166250	0.000039	...	0.5	0.275	False	0.415224	2.2050	90.0	0.9	30.0	5.0	1900-01-01 16:19:30
8	2000.0	1.0	1.0	40.0	1.0100	0.165623	0.000039	...	0.5	0.275	False	0.415224	2.6950	90.0	0.9	30.0	5.0	1900-01-01 16:21:32
9	2000.0	1.0	1.0	40.0	1.0295	0.166040	0.000039	...	0.5	0.275	False	0.415224	1.9600	90.0	0.9	30.0	5.0	1900-01-01 16:23:23
10	2000.0	1.0	1.0	40.0	1.0540	0.166372	0.000039	...	0.5	0.275	False	0.415224	3.1850	90.0	0.9	30.0	5.0	1900-01-01 16:25:12
11	2000.0	1.0	1.0	40.0	1.0195	0.166743	0.000038	...	0.5	0.275	False	0.415224	3.3075	90.0	0.9	30.0	5.0	1900-01-01 16:26:59
12	2000.0	1.0	1.0	40.0	1.0085	0.166322	0.000039	...	0.5	0.275	False	0.415224	1.5925	90.0	0.9	30.0	5.0	1900-01-01 16:28:43
13	2000.0	1.0	1.0	40.0	1.0710	0.166550	0.000039	...	0.5	0.275	False	0.415224	3.7975	90.0	0.9	30.0	5.0	1900-01-01 16:30:27
14	2000.0	1.0	1.0	40.0	1.0265	0.166740	0.000038	...	0.5	0.275	False	0.415224	3.6750	90.0	0.9	30.0	5.0	1900-01-01 16:32:09
15	2000.0	1.0	1.0	40.0	1.0715	0.166051	0.000039	...	0.5	0.275	False	0.415224	2.0825	90.0	0.9	30.0	5.0	1900-01-01 16:33:49
16	2000.0	1.0	1.0	40.0	1.0500	0.166400	0.000039	...	0.5	0.275	False	0.415224	3.0625	90.0	0.9	30.0	5.0	1900-01-01 16:35:30
17	2000.0	1.0	1.0	40.0	1.0620	0.166075	0.000039	...	0.5	0.275	False	0.415224	3.4300	90.0	0.9	30.0	5.0	1900-01-01 16:37:11
18	2000.0	1.0	1.0	40.0	1.0155	0.165926	0.000040	...	0.5	0.275	False	0.415224	2.9400	90.0	0.9	30.0	5.0	1900-01-01 16:38:52
19	2000.0	1.0	1.0	40.0	1.0610	0.165490	0.000039	...	0.5	0.275	False	0.415224	2.8175	90.0	0.9	30.0	5.0	1900-01-01 16:40:33